// netz.compress.ICompress implementation
public long Compress(string file, string zipFile)
{
long length = -1;
FileStream ifs = null;
FileStream ofs = null;
try
{
ifs = File.Open(file, FileMode.Open, FileAccess.Read, FileShare.Read);
ofs = File.Open(zipFile, FileMode.Create, FileAccess.Write, FileShare.None);
DeflateStream dos = new DeflateStream(ofs, CompressionMode.Compress, true);
byte[] buff = new byte[ifs.Length];
while(true)
{
int r = ifs.Read(buff, 0, buff.Length);
if(r <= 0) break;
dos.Write(buff, 0, r);
}
dos.Flush();
dos.Close();
length = ofs.Length;
}
finally
{
if(ifs != null) ifs.Close();
if(ofs != null) ofs.Close();
}
return length;
}
static void DecompressProfile(string srcfile, string dstfile)
{
try
{
Console.WriteLine("Reading source...");
byte[] x = File.ReadAllBytes(srcfile); // load all bytes
byte[] y = new byte[x.Length-0x1A];
Console.WriteLine("Copying array...");
Array.Copy(x, 0x1A, y, 0, y.Length); // skip hash (0x14) + uncompresed size (0x4) + zlib compression type (0x2) = 0x1A
Console.WriteLine("Initializing streams...");
DeflateStream defStream = new DeflateStream(new MemoryStream(y),CompressionMode.Decompress);
FileStream fs = File.Create(dstfile);
Console.WriteLine("Decompressing data to destination...");
defStream.CopyTo(fs);
Console.WriteLine("Decompression done.\n" + dstfile);
defStream.Close();
fs.Close();
}
catch(Exception ex)
{
Console.WriteLine(ex.GetType().Name + " | " + ex.Message);
Console.Beep();
}
return;
}
public static async Task OverlappingFlushAsync_DuringFlushAsync()
{
byte[] buffer = null;
string testFilePath = gzTestFile("GZTestDocument.pdf");
using (var origStream = await LocalMemoryStream.readAppFileAsync(testFilePath))
{
buffer = origStream.ToArray();
}
using (var writeStream = new ManualSyncMemoryStream(false))
using (var zip = new DeflateStream(writeStream, CompressionMode.Compress))
{
Task task = null;
try
{
writeStream.manualResetEvent.Set();
await zip.WriteAsync(buffer, 0, buffer.Length);
writeStream.manualResetEvent.Reset();
writeStream.WriteHit = false;
task = zip.FlushAsync();
Assert.True(writeStream.WriteHit);
Assert.Throws<InvalidOperationException>(() => { zip.FlushAsync(); }); // "overlapping flushes"
}
finally
{
// Unblock Async operations
writeStream.manualResetEvent.Set();
// The original WriteAsync should be able to complete
Assert.True(task.Wait(100 * 500));
}
}
}
/// <summary>
/// Compresses the response stream if the browser allows it.
/// </summary>
private static MemoryStream CompressResponse(Stream responseStream, HttpApplication app, string key)
{
MemoryStream dataStream = new MemoryStream();
StreamCopy(responseStream, dataStream);
responseStream.Dispose();
byte[] buffer = dataStream.ToArray();
dataStream.Dispose();
MemoryStream ms = new MemoryStream();
Stream compress = null;
if (IsEncodingAccepted(DEFLATE))
{
compress = new DeflateStream(ms, CompressionMode.Compress);
app.Application.Add(key + "enc", DEFLATE);
}
else if (IsEncodingAccepted(GZIP))
{
compress = new GZipStream(ms, CompressionMode.Compress);
app.Application.Add(key + "enc", DEFLATE);
}
compress.Write(buffer, 0, buffer.Length);
compress.Dispose();
return ms;
}
public static void Compress(FileInfo fi)
{
// Get the stream of the source file.
using (FileStream inFile = fi.OpenRead())
{
// Prevent compressing hidden and already compressed files.
if ((File.GetAttributes(fi.FullName) & FileAttributes.Hidden)
!= FileAttributes.Hidden & fi.Extension != ".cmp")
{
// Create the compressed file.
using (FileStream outFile =
File.Create(fi.FullName + ".cmp"))
{
using (DeflateStream Compress =
new DeflateStream(outFile,
CompressionMode.Compress))
{
// Copy the source file into
// the compression stream.
inFile.CopyTo(Compress);
Console.WriteLine("Compressed {0} from {1} to {2} bytes.",
fi.Name, fi.Length.ToString(), outFile.Length.ToString());
}
}
}
}
}
public static void Decompress(FileInfo fi)
{
// Get the stream of the source file.
using (FileStream inFile = fi.OpenRead())
{
// Get original file extension,
// for example "doc" from report.doc.cmp.
string curFile = fi.FullName;
string origName = curFile.Remove(curFile.Length
- fi.Extension.Length);
//Create the decompressed file.
using (FileStream outFile = File.Create(origName))
{
using (DeflateStream Decompress = new DeflateStream(inFile,
CompressionMode.Decompress))
{
// Copy the decompression stream
// into the output file.
Decompress.CopyTo(outFile);
Console.WriteLine("Decompressed: {0}", fi.Name);
}
}
}
}
void ProcessNativeResources(bool compress)
{
var unprocessedNameMatch = new Regex(@"^(.*\.)?costura(32|64)\.", RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
var processedNameMatch = new Regex(@"^costura(32|64)\.", RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
foreach (var resource in ModuleDefinition.Resources.OfType<EmbeddedResource>())
{
var match = unprocessedNameMatch.Match(resource.Name);
if (match.Success)
{
resource.Name = resource.Name.Substring(match.Groups[1].Length).ToLowerInvariant();
hasUnmanaged = true;
}
if (processedNameMatch.IsMatch(resource.Name))
{
using (var stream = resource.GetResourceStream())
{
if (compress && resource.Name.EndsWith(".zip"))
{
using (var compressStream = new DeflateStream(stream, CompressionMode.Decompress))
{
checksums.Add(resource.Name, CalculateChecksum(compressStream));
}
}
else
{
checksums.Add(resource.Name, CalculateChecksum(stream));
}
}
}
}
}
public static byte[] Decompress(byte[] data)
{
try
{
string result = string.Empty;
byte[] buffer = { };
MemoryStream ms = new MemoryStream(data);
Stream s = new DeflateStream(ms, CompressionMode.Decompress);
int len = 4096;
while (true)
{
int oldLen = buffer.Length;
Array.Resize(ref buffer, oldLen + len);
int size = s.Read(buffer, oldLen, len);
if (size != len)
{
Array.Resize(ref buffer, buffer.Length - (len - size));
break;
}
if (size <= 0)
break;
}
s.Close();
return buffer;
}
catch
{
return null;
}
}
public void BaseStream2()
{
var ms = new MemoryStream();
var zip = new DeflateStream(ms, CompressionMode.Decompress);
Assert.Same(zip.BaseStream, ms);
ms.Dispose();
}
public void BaseStream1()
{
var writeStream = new MemoryStream();
var zip = new DeflateStream(writeStream, CompressionMode.Compress);
Assert.Same(zip.BaseStream, writeStream);
writeStream.Dispose();
}
public void CompressCanWrite()
{
var ms = new MemoryStream();
var zip = new DeflateStream(ms, CompressionMode.Compress);
Assert.True(zip.CanWrite);
zip.Dispose();
Assert.False(zip.CanWrite);
}
private string Embed(string prefix, string fullPath, bool compress)
{
var resourceName = String.Format("{0}{1}", prefix, Path.GetFileName(fullPath).ToLowerInvariant());
if (ModuleDefinition.Resources.Any(x => x.Name == resourceName))
{
LogInfo(string.Format("\tSkipping '{0}' because it is already embedded", fullPath));
return resourceName;
}
if (compress)
{
resourceName = String.Format("{0}{1}.zip", prefix, Path.GetFileName(fullPath).ToLowerInvariant());
}
LogInfo(string.Format("\tEmbedding '{0}'", fullPath));
var checksum = CalculateChecksum(fullPath);
var cacheFile = Path.Combine(cachePath, String.Format("{0}.{1}", checksum, resourceName));
var memoryStream = new MemoryStream();
if (File.Exists(cacheFile))
{
using (var fileStream = File.Open(cacheFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
fileStream.CopyTo(memoryStream);
}
}
else
{
using (var cacheFileStream = File.Open(cacheFile, FileMode.CreateNew, FileAccess.Write, FileShare.Read))
{
using (var fileStream = File.Open(fullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
if (compress)
{
using (var compressedStream = new DeflateStream(memoryStream, CompressionMode.Compress, true))
{
fileStream.CopyTo(compressedStream);
}
}
else
{
fileStream.CopyTo(memoryStream);
}
}
memoryStream.Position = 0;
memoryStream.CopyTo(cacheFileStream);
}
}
memoryStream.Position = 0;
streams.Add(memoryStream);
var resource = new EmbeddedResource(resourceName, ManifestResourceAttributes.Private, memoryStream);
ModuleDefinition.Resources.Add(resource);
return resourceName;
}
public void DecompressCanRead()
{
var ms = new MemoryStream();
var zip = new DeflateStream(ms, CompressionMode.Decompress);
Assert.True(zip.CanRead);
zip.Dispose();
Assert.False(zip.CanRead);
}
public void CanDisposeDeflateStream()
{
var ms = new MemoryStream();
var zip = new DeflateStream(ms, CompressionMode.Compress);
zip.Dispose();
// Base Stream should be null after dispose
Assert.Null(zip.BaseStream);
zip.Dispose(); // Should be a no-op
}
public static byte[] Deflate(this byte[] data)
{
using (MemoryStream output = new MemoryStream())
{
using (DeflateStream gzip = new DeflateStream(output, CompressionMode.Compress))
{
gzip.Write(data, 0, data.Length);
}
return output.ToArray();
}
}
public void Decompress()
{
var bytes = CompressToArray();
using (var memoryStream = new MemoryStream(bytes))
using (var deflateStream = new DeflateStream(memoryStream, CompressionMode.Decompress))
using (var reader = new StreamReader(deflateStream))
{
string text = reader.ReadToEnd();
Assert.AreEqual(Text, text);
}
}
public void WebRequest()
{
var bytes = GetHttpBin("http://httpbin.org/deflate");
using (var memoryStream = new MemoryStream(bytes))
using (var deflateStream = new DeflateStream(memoryStream, CompressionMode.Decompress))
using (var reader = new StreamReader(deflateStream))
{
string text = reader.ReadToEnd();
Assert.IsTrue(text.Contains("\"deflated\": true"), "Request was not compressed!");
}
}
protected override byte[] CompressToArray()
{
using (var memoryStream = new MemoryStream())
{
using (var deflateStream = new DeflateStream(memoryStream, CompressionMode.Compress))
using (var writer = new StreamWriter(deflateStream))
{
writer.Write(Text);
}
return memoryStream.ToArray();
}
}
public void BuildAxisOrderBitArray()
{
if (!File.Exists(EpsgAccessDatabase))
throw new IgnoreException("Epsg Access Database not found");
var ba = new BitArray(32768);
using (var cn = new System.Data.OleDb.OleDbConnection(
string.Format("Provider=Microsoft.Jet.OLEDB.4.0; Data Source={0};", EpsgAccessDatabase)))
{
cn.Open();
var cmd = cn.CreateCommand();
cmd.CommandText = Sql;
using (var dr = cmd.ExecuteReader())
{
if (dr != null)
while (dr.Read())
{
var code = dr.GetInt32(0);
if (code > 32767) continue;
ba[code] = true;
}
}
}
string enc;
var buffer = new byte[4096];
ba.CopyTo(buffer, 0);
#if NET45
using (var bufferStream = new MemoryStream(buffer))
{
using (var compressedStream = new MemoryStream())
{
using (var s = new DeflateStream(compressedStream, CompressionMode.Compress))
{
bufferStream.CopyTo(s);
compressedStream.Flush();
}
enc = Convert.ToBase64String(compressedStream.ToArray());
Console.WriteLine("Compressed");
WriteBlocks(enc);
}
}
#endif
enc = Convert.ToBase64String(buffer);
Console.WriteLine("\nUncompressed");
WriteBlocks(enc);
Console.WriteLine("\nByte array");
WriteBytes(buffer, 20);
}
public async Task ModifyBaseStream()
{
var ms = await LocalMemoryStream.readAppFileAsync(gzTestFile("GZTestDocument.txt.gz"));
var newMs = StripHeaderAndFooter.Strip(ms);
var zip = new DeflateStream(newMs, CompressionMode.Decompress);
int size = 1024;
byte[] bytes = new byte[size];
zip.BaseStream.Read(bytes, 0, size); // This will throw if the underlying stream is not writable as expected
zip.BaseStream.Position = 0;
await zip.BaseStream.ReadAsync(bytes, 0, size);
}
static void CompressProfile(string srcfile, string dstfile)
{
try
{
Console.WriteLine("Reading source...");
byte[] x = File.ReadAllBytes(srcfile);
int usize = x.Length;
Console.WriteLine("Initializing memory stream...");
MemoryStream ms = new MemoryStream();
// write uncompressed size as big endian
ms.WriteByte((byte)((usize>>24) & 0xFF));
ms.WriteByte((byte)((usize>>16) & 0xFF));
ms.WriteByte((byte)((usize>>8) & 0xFF));
ms.WriteByte((byte)(usize & 0xFF));
// then, compressed data
// these two bytes are part of zlib standard, but aren't supported by DeflateStream
ms.WriteByte(0x78);
ms.WriteByte(0x9C);
Console.WriteLine("Compressing data...");
MemoryStream compData = new MemoryStream();
DeflateStream ds = new DeflateStream(compData, CompressionMode.Compress);
ds.Write(x, 0, x.Length);
ds.Close();
ms.Write(compData.ToArray(), 0, compData.ToArray().Length);
// Adler32 checksum as big endian - also not supported by DeflateStream, but required by zlib standard
int checksum = GetAdler32(x);
ms.WriteByte((byte)((checksum>>24) & 0xFF));
ms.WriteByte((byte)((checksum>>16) & 0xFF));
ms.WriteByte((byte)((checksum>>8) & 0xFF));
ms.WriteByte((byte)(checksum & 0xFF));
// start filestream
Console.WriteLine("Creating file stream...");
FileStream fs = File.Create(dstfile);
// write hash
fs.Write(new SHA1CryptoServiceProvider().ComputeHash(ms.ToArray()), 0, 0x14);
// write usize + compressed data
fs.Write(ms.ToArray(), 0, ms.ToArray().Length);
Console.WriteLine("Compression done.\n" + dstfile);
fs.Close();
ms.Close();
compData.Close();
}
catch(Exception ex)
{
Console.WriteLine(ex.GetType().Name + " | " + ex.Message);
Console.Beep();
}
return;
}
public static string UnZipStr(byte[] input)
{
using (MemoryStream inputStream = new MemoryStream(input))
{
using (DeflateStream gzip =
new DeflateStream(inputStream, CompressionMode.Decompress))
{
using (StreamReader reader =
new StreamReader(gzip, System.Text.Encoding.UTF8))
{
return reader.ReadToEnd();
}
}
}
}
public void LeaveOpenTrue()
{
using (var memoryStream = new MemoryStream())
{
using (var deflateStream = new DeflateStream(memoryStream, CompressionMode.Compress, true))
using (var writer = new StreamWriter(deflateStream))
{
writer.Write(Text);
}
using (var deflateStream = new DeflateStream(memoryStream, CompressionMode.Compress, true))
using (var writer = new StreamWriter(deflateStream))
{
writer.Write(Text);
}
}
}
public static byte[] ZipStr(String str)
{
using (MemoryStream output = new MemoryStream())
{
using (DeflateStream gzip =
new DeflateStream(output, CompressionMode.Compress))
{
using (StreamWriter writer =
new StreamWriter(gzip, System.Text.Encoding.UTF8))
{
writer.Write(str);
}
}
return output.ToArray();
}
}
public static byte[] Compress(byte[] data)
{
try
{
MemoryStream ms = new MemoryStream();
Stream s = new DeflateStream(ms, CompressionMode.Compress);
s.Write(data, 0, data.Length);
s.Close();
return ms.ToArray();
}
catch
{
return null;
}
}
public static byte[] Compress(byte[] data)
{
try
{
//Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.RealTime;
MemoryStream ms = new MemoryStream();
Stream s = new DeflateStream(ms, CompressionMode.Compress);
s.Write(data, 0, data.Length);
s.Close();
//Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.Normal;
return ms.ToArray();
}
catch
{
//Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.Normal;
return null;
}
}
public static byte[] Inflate(this byte[] data)
{
using (MemoryStream input = new MemoryStream(data))
{
input.Write(data, 0, data.Length);
input.Position = 0;
using (DeflateStream gzip = new DeflateStream(input, CompressionMode.Decompress))
{
using (MemoryStream output = new MemoryStream())
{
var buffer = new byte[ushort.MaxValue]; int count;
while ((count = gzip.Read(buffer, 0, buffer.Length)) > 0)
{
output.Write(buffer, 0, count);
}
return output.ToArray();
}
}
}
}
public static async Task OverlappingReadAsync()
{
byte[] buffer = new byte[32];
string testFilePath = gzTestFile("GZTestDocument.pdf.gz");
using (var readStream = await ManualSyncMemoryStream.GetStreamFromFileAsync(testFilePath, false, true))
using (var unzip = new DeflateStream(readStream, CompressionMode.Decompress, true))
{
Task task = null;
try
{
task = unzip.ReadAsync(buffer, 0, 32);
Assert.True(readStream.ReadHit);
Assert.Throws<InvalidOperationException>(() => { unzip.ReadAsync(buffer, 0, 32); }); // "overlapping read"
}
finally
{
// Unblock Async operations
readStream.manualResetEvent.Set();
// The original ReadAsync should be able to complete
Assert.True(task.Wait(100 * 500));
}
}
}
/// <summary>
/// Encodes the specified data.
/// </summary>
public byte[] Encode(byte[] data, PdfFlateEncodeMode mode)
{
MemoryStream ms = new MemoryStream();
// DeflateStream/GZipStream does not work immediately and I have not the leisure to work it out.
// So I keep on using SharpZipLib even with .NET 2.0.
#if NET_ZIP
// See http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=97064
//
// Excerpt from the RFC 1950 specs for first byte:
//
// CMF (Compression Method and flags)
// This byte is divided into a 4-bit compression method and a 4-
// bit information field depending on the compression method.
//
// bits 0 to 3 CM Compression method
// bits 4 to 7 CINFO Compression info
//
// CM (Compression method)
// This identifies the compression method used in the file. CM = 8
// denotes the "deflate" compression method with a window size up
// to 32K. This is the method used by gzip and PNG (see
// references [1] and [2] in Chapter 3, below, for the reference
// documents). CM = 15 is reserved. It might be used in a future
// version of this specification to indicate the presence of an
// extra field before the compressed data.
//
// CINFO (Compression info)
// For CM = 8, CINFO is the base-2 logarithm of the LZ77 window
// size, minus eight (CINFO=7 indicates a 32K window size). Values
// of CINFO above 7 are not allowed in this version of the
// specification. CINFO is not defined in this specification for
// CM not equal to 8.
ms.WriteByte(0x78);
// Excerpt from the RFC 1950 specs for second byte:
//
// FLG (FLaGs)
// This flag byte is divided as follows:
//
// bits 0 to 4 FCHECK (check bits for CMF and FLG)
// bit 5 FDICT (preset dictionary)
// bits 6 to 7 FLEVEL (compression level)
//
// The FCHECK value must be such that CMF and FLG, when viewed as
// a 16-bit unsigned integer stored in MSB order (CMF*256 + FLG),
// is a multiple of 31.
//
// FDICT (Preset dictionary)
// If FDICT is set, a DICT dictionary identifier is present
// immediately after the FLG byte. The dictionary is a sequence of
// bytes which are initially fed to the compressor without
// producing any compressed output. DICT is the Adler-32 checksum
// of this sequence of bytes (see the definition of ADLER32
// below). The decompressor can use this identifier to determine
// which dictionary has been used by the compressor.
//
// FLEVEL (Compression level)
// These flags are available for use by specific compression
// methods. The "deflate" method (CM = 8) sets these flags as
// follows:
//
// 0 - compressor used fastest algorithm
// 1 - compressor used fast algorithm
// 2 - compressor used default algorithm
// 3 - compressor used maximum compression, slowest algorithm
//
// The information in FLEVEL is not needed for decompression; it
// is there to indicate if recompression might be worthwhile.
ms.WriteByte(0x49);
DeflateStream zip = new DeflateStream(ms, CompressionMode.Compress, true);
zip.Write(data, 0, data.Length);
zip.Close();
#else
int level = Deflater.DEFAULT_COMPRESSION;
switch (mode)
{
case PdfFlateEncodeMode.BestCompression:
level = Deflater.BEST_COMPRESSION;
break;
case PdfFlateEncodeMode.BestSpeed:
level = Deflater.BEST_SPEED;
break;
}
DeflaterOutputStream zip = new DeflaterOutputStream(ms, new Deflater(level, false));
zip.Write(data, 0, data.Length);
zip.Finish();
#endif
#if !NETFX_CORE && !UWP
ms.Capacity = (int)ms.Length;
return(ms.GetBuffer());
#else
return(ms.ToArray());
#endif
}
public void Constructor_Null()
{
DeflateStream ds = new DeflateStream(null, CompressionMode.Compress);
}
public override Task <BaseWebSocketClient> ConnectAsync(Uri uri)
{
this.StreamDecompressor?.Dispose();
this.CompressedStream?.Dispose();
this.DecompressedStream?.Dispose();
this.DecompressedStream = new MemoryStream();
this.CompressedStream = new MemoryStream();
this.StreamDecompressor = new DeflateStream(this.CompressedStream, CompressionMode.Decompress);
_socket = new wss.WebSocket(uri.ToString());
_socket.OnOpen += HandlerOpen;
_socket.OnClose += HandlerClose;
_socket.OnMessage += HandlerMessage;
_socket.Connect();
return(Task.FromResult <BaseWebSocketClient>(this));
void HandlerOpen(object sender, s.EventArgs e)
=> _connect.InvokeAsync().ConfigureAwait(false).GetAwaiter().GetResult();
void HandlerClose(object sender, wss.CloseEventArgs e)
=> _disconnect.InvokeAsync(new SocketCloseEventArgs(null)
{
CloseCode = e.Code, CloseMessage = e.Reason
}).ConfigureAwait(false).GetAwaiter().GetResult();
void HandlerMessage(object sender, wss.MessageEventArgs e)
{
var msg = "";
if (e.IsBinary)
{
if (e.RawData[0] == 0x78)
{
this.CompressedStream.Write(e.RawData, 2, e.RawData.Length - 2);
}
else
{
this.CompressedStream.Write(e.RawData, 0, e.RawData.Length);
}
this.CompressedStream.Flush();
this.CompressedStream.Position = 0;
// partial credit to FiniteReality
// overall idea is his
// I tuned the finer details
// -Emzi
var sfix = BitConverter.ToUInt16(e.RawData, e.RawData.Length - 2);
if (sfix != ZLIB_STREAM_SUFFIX)
{
using (var zlib = new DeflateStream(this.CompressedStream, CompressionMode.Decompress, true))
zlib.CopyTo(this.DecompressedStream);
}
else
{
this.StreamDecompressor.CopyTo(this.DecompressedStream);
}
msg = UTF8.GetString(this.DecompressedStream.ToArray(), 0, (int)this.DecompressedStream.Length);
this.DecompressedStream.Position = 0;
this.DecompressedStream.SetLength(0);
this.CompressedStream.Position = 0;
this.CompressedStream.SetLength(0);
}
else
{
msg = e.Data;
}
_message.InvokeAsync(new SocketMessageEventArgs()
{
Message = msg
}).ConfigureAwait(false).GetAwaiter().GetResult();
}
}
public void CheckGetPositionProp()
{
MemoryStream backing = new MemoryStream(compressed_data);
DeflateStream decompressing = new DeflateStream(backing, CompressionMode.Decompress);
long position = decompressing.Position;
}
public static void Users(int userCount, string dePath, string defpwd, object sender, DoWorkEventArgs dwea)
{
int successCount = 0;
int iterationCount = 0;
int consecutiveFailures = 0;
var bw = sender as BackgroundWorker;
var de = new DirectoryEntry(dePath);
// without this AuthenticationType property, one of my dev boxes would
// prompt for smart card during "setpassword" call
// seemed to be from re-binding to LDAP using SSL
// see: https://social.technet.microsoft.com/Forums/scriptcenter/en-US/d2c9347c-cfd6-44a8-8baf-7c5202999177/adsi-setpassword-requests-smart-card-pin?forum=ITCG
// https://gitlab.i.caplette.org/open-source/labbuilder/issues/1
de.AuthenticationType = AuthenticationTypes.Secure | AuthenticationTypes.Signing | AuthenticationTypes.Sealing;
bw.ReportProgress(0, "Initializing...");
var r = new Random();
ts.TraceEvent(TraceEventType.Verbose, 0, "Creating {0} users...", userCount);
// load and decompress name list if necessary
if (names == null)
{
MemoryStream ms = null;
try
{
ms = new MemoryStream(LabBuilder.Properties.Resources.imdbActresses_txt);
using (DeflateStream ds = new DeflateStream(ms, CompressionMode.Decompress))
{
ms = null;
names = (List <string>)bf.Deserialize(ds);
}
}
finally
{
if (ms != null)
{
ms.Dispose();
}
}
ts.TraceEvent(TraceEventType.Verbose, 0, "Initialized {0:N0} possible names...", names.Count);
}
while (successCount < userCount && consecutiveFailures < maxConsecutiveFailuresCreatingUsers)
{
// check for cancellation
if (bw.CancellationPending)
{
dwea.Cancel = true;
break;
}
bw.ReportProgress((int)((double)successCount / userCount * 100), string.Format(CultureInfo.CurrentCulture, "Created {0:N0} of {1:N0} users", successCount, userCount));
iterationCount++;
// setup new user info
string name = names[r.Next(names.Count)];
string firstname;
string lastname;
string SAMAccountName;
string wholeName;
// remove parens (with contents) from name
name = Regex.Replace(name, @"\([^)]+\)", string.Empty);
// if name has comma, get firstname and last name
// TODO: Middle name processing?
if (name.Contains(','))
{
int commapos = name.IndexOf(',');
firstname = name.Substring(commapos + 1).Trim();
lastname = name.Substring(0, commapos).Trim();
}
else // no comma, no first name
{
lastname = name.Trim();
firstname = "";
}
if (firstname.Length > 0)
{
SAMAccountName = firstname + "." + lastname;
wholeName = firstname + " " + lastname;
}
else
{
SAMAccountName = lastname;
wholeName = lastname;
}
// Name may have accented chars, replace with same non-accented characters for SAMAccountName
// trick I learned from Stack OVerflow, no idea why Cyrillic
SAMAccountName = Encoding.ASCII.GetString(Encoding.GetEncoding("Cyrillic").GetBytes(SAMAccountName));
// replace all sequential non-word characters with a period
SAMAccountName = Regex.Replace(SAMAccountName, @"[^\w]+", ".");
// 20 char limit for SAMAccountName
if (SAMAccountName.Length > 20)
{
SAMAccountName = SAMAccountName.Substring(0, 20);
}
// SAMAccountName cannot end (or begin) with a dot
SAMAccountName = SAMAccountName.Trim('.');
// make sure SAMAccountName is unique (for this run of the program)
// not checking with AD for performance reasons, will just get object exists error
// and move on. (Don't optimize the uncommon cases!!)
if (usedNames.Contains(SAMAccountName))
{
continue;
}
usedNames.Add(SAMAccountName);
RandomAttributes ra = new RandomAttributes();
ts.TraceEvent(TraceEventType.Verbose, 0,
"Creating name: {0}, SAMAccountName: {1}, Title: {2}, Office Phone: {3}, Street: {4}",
name, SAMAccountName, ra.Title, ra.OfficePhone, ra.StreetAddress);
try
{
DirectoryEntry newuser = de.Children.Add
("CN=" + wholeName, "user");
newuser.Properties["samAccountName"].Value = SAMAccountName;
newuser.Properties["userPrincipalName"].Value = SAMAccountName + "@" + DirectoryEntryContext.DefaultDomain;
if (firstname.Length > 0)
{
newuser.Properties["givenName"].Value = firstname;
}
newuser.Properties["SN"].Value = lastname;
newuser.Properties["displayName"].Value = wholeName;
newuser.Properties["l"].Value = ra.City;
newuser.Properties["c"].Value = ra.Country;
newuser.Properties["department"].Value = ra.Department;
newuser.Properties["division"].Value = ra.Division;
newuser.Properties["employeeID"].Value = ra.EmployeeID;
newuser.Properties["homePhone"].Value = ra.HomePhone;
newuser.Properties["physicalDeliveryOfficeName"].Value = ra.Office;
newuser.Properties["telephoneNumber"].Value = ra.OfficePhone;
newuser.Properties["o"].Value = ra.Organization;
newuser.Properties["postalCode"].Value = ra.PostalCode;
newuser.Properties["st"].Value = ra.State;
newuser.Properties["streetAddress"].Value = ra.StreetAddress;
newuser.Properties["title"].Value = ra.Title;
newuser.Properties["extensionAttribute13"].Value = "{" + defpwd;
newuser.CommitChanges();
newuser.Invoke("SetPassword", new object[] { defpwd });
// get existing value
int val = (int)newuser.Properties["userAccountControl"].Value;
//set flags to enable account and disable password expiry
newuser.Properties["userAccountControl"].Value = (val & ~0x2) | 0x10000;
newuser.CommitChanges();
newuser.Close();
successCount++;
consecutiveFailures = 0;
}
catch (COMException ce)
{
if (!(ce.ErrorCode == unchecked ((int)0x80071392) || ce.ErrorCode == unchecked ((int)0x8007202F)))
{
// It's not "The object already exists" or "A constraint violation occurred"
// which is expected error if user exists. If user exists in another OU the
// constaint error is given.
consecutiveFailures++;
ts.TraceEvent(TraceEventType.Error, 0, "creating user {0}. {1}: {2}", name, ce.ErrorCode, ce.ToString());
}
}
catch (Exception e)
{
consecutiveFailures++;
ts.TraceEvent(TraceEventType.Error, 0, "creating user {0}: {1}", name, e.ToString());
}
}
if (bw.CancellationPending)
{
bw.ReportProgress(0, string.Format(CultureInfo.CurrentCulture, "Cancelled. Created {0:N0} users.", successCount));
}
else if (consecutiveFailures == maxConsecutiveFailuresCreatingUsers)
{
bw.ReportProgress(0, string.Format(CultureInfo.CurrentCulture, "Stopped due to too many failures. Created {0:N0} users.", successCount));
}
else
{
bw.ReportProgress(0, string.Format(CultureInfo.CurrentCulture, "Complete. Created {0:N0} users.", successCount));
}
// clean up non-memory resources
if (de != null)
{
de.Dispose();
}
}
public async Task WrapNullReturningTasksStream()
{
using (var ds = new DeflateStream(new BadWrappedStream(BadWrappedStream.Mode.ReturnNullTasks), CompressionMode.Decompress))
await Assert.ThrowsAsync <ArgumentNullException>(() => ds.ReadAsync(new byte[1024], 0, 1024));
}
public static async Task <IEnumerable <TBUserNote> > GetUserNotesAsync(IWebAgent webAgent, string subName)
{
var reqResponse = await webAgent.Get(string.Format(ToolBoxUserNotesWiki, subName)).ConfigureAwait(false);
var response = JObject.Parse(reqResponse["data"]["content_md"].Value <string>());
int version = response["ver"].Value <int>();
if (version < 6)
{
throw new ToolBoxUserNotesException("Unsupported ToolBox version");
}
string[] mods = response["constants"]["users"].Values <string>().ToArray();
string[] warnings = response["constants"]["warnings"].Values <string>().ToArray();
try
{
var data = Convert.FromBase64String(response["blob"].Value <string>());
string uncompressed;
using (System.IO.MemoryStream compressedStream = new System.IO.MemoryStream(data))
{
compressedStream.ReadByte();
compressedStream.ReadByte(); //skips first to bytes to fix zlib block size
using (DeflateStream blobStream = new DeflateStream(compressedStream, CompressionMode.Decompress))
{
using (var decompressedReader = new System.IO.StreamReader(blobStream))
{
uncompressed = decompressedReader.ReadToEnd();
}
}
}
JObject users = JObject.Parse(uncompressed);
List <TBUserNote> toReturn = new List <TBUserNote>();
foreach (KeyValuePair <string, JToken> user in users)
{
var x = user.Value;
foreach (JToken note in x["ns"].Children())
{
TBUserNote uNote = new TBUserNote()
{
AppliesToUsername = user.Key,
SubName = subName,
SubmitterIndex = note["m"].Value <int>(),
Submitter = mods[note["m"].Value <int>()],
NoteTypeIndex = note["w"].Value <int>(),
NoteType = warnings[note["w"].Value <int>()],
Message = note["n"].Value <string>(),
Timestamp = UnixTimeStamp.UnixTimeStampToDateTime(note["t"].Value <long>()),
Url = UnsquashLink(subName, note["l"].ValueOrDefault <string>())
};
toReturn.Add(uNote);
}
}
return(toReturn);
}
catch (Exception e)
{
throw new ToolBoxUserNotesException("An error occured while processing Usernotes wiki. See inner exception for details", e);
}
}
private static Stream Decompress(Stream stream)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
try
{
stream.Seek(0, SeekOrigin.Begin);
int type = (int)Varint.GetUInt64(stream);
if (type == (int)ConvertCompressionAlgorithm.None)
{
return(new RangeStream(stream));
}
else if (type == (int)ConvertCompressionAlgorithm.Deflate)
{
RecyclableMemoryStream deflateBufferStream = null;
try
{
deflateBufferStream = new RecyclableMemoryStream(_bufferManager);
using (var deflateStream = new DeflateStream(stream, CompressionMode.Decompress))
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
{
int length;
while ((length = deflateStream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
deflateBufferStream.Write(safeBuffer.Value, 0, length);
if (deflateBufferStream.Length > 1024 * 1024 * 256)
{
throw new Exception("too large");
}
}
}
deflateBufferStream.Seek(0, SeekOrigin.Begin);
return(deflateBufferStream);
}
catch (Exception)
{
if (deflateBufferStream != null)
{
deflateBufferStream.Dispose();
}
throw;
}
}
else
{
throw new ArgumentException("ArgumentException");
}
}
catch (Exception e)
{
if (stream != null)
{
stream.Dispose();
}
throw new ArgumentException(e.Message, e);
}
}
private static Stream Compress(Stream stream)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
try
{
var dic = new Dictionary <byte, Stream>();
try
{
stream.Seek(0, SeekOrigin.Begin);
RecyclableMemoryStream deflateBufferStream = null;
try
{
deflateBufferStream = new RecyclableMemoryStream(_bufferManager);
using (var deflateStream = new DeflateStream(deflateBufferStream, CompressionMode.Compress, true))
using (var safeBuffer = _bufferManager.CreateSafeBuffer(1024 * 4))
{
int length;
while ((length = stream.Read(safeBuffer.Value, 0, safeBuffer.Value.Length)) > 0)
{
deflateStream.Write(safeBuffer.Value, 0, length);
}
}
deflateBufferStream.Seek(0, SeekOrigin.Begin);
dic.Add((byte)ConvertCompressionAlgorithm.Deflate, deflateBufferStream);
}
catch (Exception)
{
if (deflateBufferStream != null)
{
deflateBufferStream.Dispose();
}
}
}
catch (Exception)
{
}
dic.Add((byte)ConvertCompressionAlgorithm.None, stream);
var list = dic.ToList();
list.Sort((x, y) =>
{
int c = x.Value.Length.CompareTo(y.Value.Length);
if (c != 0)
{
return(c);
}
return(x.Key.CompareTo(y.Key));
});
for (int i = 1; i < list.Count; i++)
{
list[i].Value.Dispose();
}
var headerStream = new RecyclableMemoryStream(_bufferManager);
Varint.SetUInt64(headerStream, list[0].Key);
return(new UniteStream(headerStream, list[0].Value));
}
catch (Exception ex)
{
if (stream != null)
{
stream.Dispose();
}
throw new ArgumentException(ex.Message, ex);
}
}
public QRCodeData(byte[] rawData, Compression compressMode)
{
var bytes = new List <byte>(rawData);
//Decompress
if (compressMode == Compression.Deflate)
{
using (var input = new MemoryStream(bytes.ToArray()))
{
using (var output = new MemoryStream())
{
using (var dstream = new DeflateStream(input, CompressionMode.Decompress))
{
//Stream4Methods.CopyTo(dstream, output);
}
bytes = new List <byte>(output.ToArray());
}
}
}
else if (compressMode == Compression.GZip)
{
using (var input = new MemoryStream(bytes.ToArray()))
{
using (var output = new MemoryStream())
{
using (var dstream = new GZipStream(input, CompressionMode.Decompress))
{
//Stream4Methods.CopyTo(dstream, output);
}
bytes = new List <byte>(output.ToArray());
}
}
}
if (bytes[0] != 0x51 || bytes[1] != 0x52 || bytes[2] != 0x52)
{
throw new Exception("Invalid raw data file. Filetype doesn't match \"QRR\".");
}
//Set QR code version
var sideLen = (int)bytes[4];
bytes.RemoveRange(0, 5);
this.Version = (sideLen - 21 - 8) / 4 + 1;
//Unpack
var modules = new Queue <bool>(8 * bytes.Count);
foreach (var b in bytes)
{
var bArr = new BitArray(new byte[] { b });
for (int i = 7; i >= 0; i--)
{
modules.Enqueue((b & (1 << i)) != 0);
}
}
//Build module matrix
this.ModuleMatrix = new List <BitArray>(sideLen);
for (int y = 0; y < sideLen; y++)
{
this.ModuleMatrix.Add(new BitArray(sideLen));
for (int x = 0; x < sideLen; x++)
{
this.ModuleMatrix[y][x] = modules.Dequeue();
}
}
}
/// <summary>
/// Decodes the specified data.
/// </summary>
public override byte[] Decode(byte[] data, FilterParms parms)
{
MemoryStream msInput = new MemoryStream(data);
MemoryStream msOutput = new MemoryStream();
#if NET_ZIP
// See http://connect.microsoft.com/VisualStudio/feedback/ViewFeedback.aspx?FeedbackID=97064
// It seems to work when skipping the first two bytes.
byte header; // 0x30 0x59
header = (byte)msInput.ReadByte();
//Debug.Assert(header == 48);
header = (byte)msInput.ReadByte();
//Debug.Assert(header == 89);
DeflateStream zip = new DeflateStream(msInput, CompressionMode.Decompress, true);
int cbRead;
byte[] abResult = new byte[1024];
do
{
cbRead = zip.Read(abResult, 0, abResult.Length);
if (cbRead > 0)
{
msOutput.Write(abResult, 0, cbRead);
}
}while (cbRead > 0);
zip.Close();
msOutput.Flush();
if (msOutput.Length >= 0)
{
msOutput.Capacity = (int)msOutput.Length;
return(msOutput.GetBuffer());
}
return(null);
#else
InflaterInputStream iis = new InflaterInputStream(msInput, new Inflater(false));
int cbRead;
byte[] abResult = new byte[32768];
do
{
cbRead = iis.Read(abResult, 0, abResult.Length);
if (cbRead > 0)
{
msOutput.Write(abResult, 0, cbRead);
}
}while (cbRead > 0);
#if UWP
iis.Dispose();
#else
iis.Close();
#endif
msOutput.Flush();
if (msOutput.Length >= 0)
{
#if NETFX_CORE || UWP
return(msOutput.ToArray());
#else
msOutput.Capacity = (int)msOutput.Length;
return(msOutput.GetBuffer());
#endif
}
return(null);
#endif
}
public void CanDisposeBaseStream()
{
var ms = new MemoryStream();
var zip = new DeflateStream(ms, CompressionMode.Compress);
ms.Dispose(); // This would throw if this was invalid
}
internal void HandlePackage(Package message, PlayerNetworkSession playerSession)
{
try
{
if (message == null)
{
return;
}
if (typeof(UnknownPackage) == message.GetType())
{
UnknownPackage packet = (UnknownPackage)message;
if (Log.IsDebugEnabled)
{
Log.Warn($"Received unknown package 0x{message.Id:X2}\n{Package.HexDump(packet.Message)}");
}
message.PutPool();
return;
}
if (typeof(McpeWrapper) == message.GetType())
{
McpeWrapper batch = (McpeWrapper)message;
var messages = new List <Package>();
// Get bytes
byte[] payload = batch.payload;
if (playerSession.CryptoContext != null && playerSession.CryptoContext.UseEncryption)
{
payload = CryptoUtils.Decrypt(payload, playerSession.CryptoContext);
}
// Decompress bytes
MemoryStream stream = new MemoryStream(payload);
if (stream.ReadByte() != 0x78)
{
throw new InvalidDataException("Incorrect ZLib header. Expected 0x78 0x9C");
}
stream.ReadByte();
using (var defStream2 = new DeflateStream(stream, CompressionMode.Decompress, false))
{
// Get actual package out of bytes
using (MemoryStream destination = MiNetServer.MemoryStreamManager.GetStream())
{
defStream2.CopyTo(destination);
destination.Position = 0;
NbtBinaryReader reader = new NbtBinaryReader(destination, true);
while (destination.Position < destination.Length)
{
//int len = reader.ReadInt32();
int len = BatchUtils.ReadLength(destination);
byte[] internalBuffer = reader.ReadBytes(len);
//if (Log.IsDebugEnabled)
// Log.Debug($"0x{internalBuffer[0]:x2}\n{Package.HexDump(internalBuffer)}");
messages.Add(PackageFactory.CreatePackage(internalBuffer[0], internalBuffer, "mcpe") ??
new UnknownPackage(internalBuffer[0], internalBuffer));
}
if (destination.Length > destination.Position)
{
throw new Exception("Have more data");
}
}
}
foreach (var msg in messages)
{
msg.DatagramSequenceNumber = batch.DatagramSequenceNumber;
msg.Reliability = batch.Reliability;
msg.ReliableMessageNumber = batch.ReliableMessageNumber;
msg.OrderingChannel = batch.OrderingChannel;
msg.OrderingIndex = batch.OrderingIndex;
HandlePackage(msg, playerSession);
}
message.PutPool();
return;
}
MiNetServer.TraceReceive(message);
if (CryptoContext != null && CryptoContext.UseEncryption)
{
MiNetServer.FastThreadPool.QueueUserWorkItem(delegate()
{
HandlePackage(MessageHandler, message as Package);
message.PutPool();
});
}
else
{
HandlePackage(MessageHandler, message);
message.PutPool();
}
}
catch (Exception e)
{
Log.Error("Package handling", e);
throw;
}
}
private void AnalyseFFXIVPacket(byte[] payload)
{
try {
while (true)
{
if (payload.Length < 4)
{
break;
}
var type = BitConverter.ToUInt16(payload, 0);
if (type == 0x0000 || type == 0x5252)
{
if (payload.Length < 28)
{
break;
}
var length = BitConverter.ToInt32(payload, 24);
if (length <= 0 || payload.Length < length)
{
break;
}
using (var messages = new MemoryStream(payload.Length))
{
using (var stream = new MemoryStream(payload, 0, length))
{
stream.Seek(40, SeekOrigin.Begin);
if (payload[33] == 0x00)
{
stream.CopyTo(messages);
}
else
{
stream.Seek(2, SeekOrigin.Current); // .Net DeflateStream 버그 (앞 2바이트 강제 무시)
using (var z = new DeflateStream(stream, CompressionMode.Decompress))
{
z.CopyTo(messages);
}
}
}
messages.Seek(0, SeekOrigin.Begin);
var messageCount = BitConverter.ToUInt16(payload, 30);
for (var i = 0; i < messageCount; i++)
{
try
{
var buffer = new byte[4];
var read = messages.Read(buffer, 0, 4);
if (read < 4)
{
Log.E("l-analyze-error-length", read, i, messageCount);
break;
}
var messageLength = BitConverter.ToInt32(buffer, 0);
var message = new byte[messageLength];
messages.Seek(-4, SeekOrigin.Current);
messages.Read(message, 0, messageLength);
HandleMessage(message);
}
catch (Exception ex)
{
Log.Ex(ex, "l-analyze-error-general");
}
}
}
if (length < payload.Length)
{
// 더 처리해야 할 패킷이 남아 있음
payload = payload.Skip(length).ToArray();
continue;
}
}
else
{
// 앞쪽이 잘려서 오는 패킷 workaround
// 잘린 패킷 1개는 버리고 바로 다음 패킷부터 찾기...
// TODO: 버리는 패킷 없게 제대로 수정하기
for (var offset = 0; offset < payload.Length - 2; offset++)
{
var possibleType = BitConverter.ToUInt16(payload, offset);
if (possibleType == 0x5252)
{
payload = payload.Skip(offset).ToArray();
AnalyseFFXIVPacket(payload);
break;
}
}
}
break;
}
}
catch (Exception ex)
{
Log.Ex(ex, "l-analyze-error");
}
}
private void dataGridView1_SelectionChanged(object sender, EventArgs e)
{
int tt = 0;
try
{
if (dataGridView1.SelectedRows.Count > 0)
{
if (dataGridView1.SelectedRows.Count > 0)
{
resetui();
string seq = dataGridView1.SelectedRows[0].Cells["seq"].Value.ToString();
if (dic_ack_httpsession.ContainsKey(seq))
{
httpsession osession = dic_ack_httpsession[seq];
var requestBody = dataGridView1.SelectedRows[0].Cells["RAW"].Value.ToString();
this.textview_requst.SetText(requestBody);
if (!string.IsNullOrWhiteSpace(requestBody))
{
try
{
jsonViewerRequest.Json = requestBody;
}
catch
{
}
}
if (osession.responseraw == null)
{
debugmsg("没有数据");
return;
}
byte[] dataall = osession.responseraw.ToArray();
this.hexview_responce.body = dataall;
this.hexview_request.body = osession.sendraw.ToArray();
int tag = 0;
for (int i = 0; i < dataall.Length - 3; i++)
{
if (dataall[i] == 0x0D && dataall[i + 1] == 0x0A && dataall[i + 2] == 0x0D && dataall[i + 3] == 0x0A)
{
tag = i;
break;
}
}
byte[] headbyte = new byte[tag + 4];
Array.Copy(dataall, 0, headbyte, 0, tag + 4);
byte[] recbyte = new byte[dataall.Length - headbyte.Length];
Array.Copy(dataall, tag + 4, recbyte, 0, recbyte.Length);
string headstring = Encoding.ASCII.GetString(headbyte).Trim(new char[] { '\0' });
if (headstring.IndexOf("chunk") >= 0)
{
recbyte = Chunk.doUnchunk(recbyte);
}
// this.hexview_responce.body = headbyte;
System.IO.Stream responseStream = new System.IO.MemoryStream(recbyte);
string compress = getencompress(headstring);
if (compress == "gzip")
{
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
}
else if (compress == "deflate")// if (headstring.ToLower().Contains("deflate"))
{
responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
}
if (isimage(headstring)) //图形
{
this.imgview.body = recbyte;
}
else
{
string encstring = getencfromhead(headstring);
if (encstring == "")
{
string prs = System.Text.Encoding.ASCII.GetString(recbyte);
encstring = getencfromhead(prs);
}
if (encstring == "")
{
encstring = "utf-8";
}
Encoding enc = Encoding.GetEncoding(encstring);
#region
// HttpWatch.Control.ByteCollection bytecollection = new HttpWatch.Control.ByteCollection();
// while (true)
// {
// byte[] buf = new byte[1024];
// int l = responseStream.Read(buf, 0, buf.Length);
// if (l <= 0) break;
// if (l < buf.Length)
// {
// byte[] n = new byte[l];
// Array.Copy(buf, 0, n, 0, n.Length);
// bytecollection.AddRange(n);
// }
// else
// {
// bytecollection.AddRange(buf);
// }
// }
// //textview_responce.m_headers = new Fiddler.HTTPResponseHeaders(enc);
//// textview_responce..
// //textview_responce.m_headers.Add("Content-Type", "text/xml; charset=" + encstring);
// textview_responce.body = bytecollection.GetBytes();
#endregion
System.IO.StreamReader sr = new System.IO.StreamReader(responseStream, enc);
string html = sr.ReadToEnd();
//string body = html.Substring(html.IndexOf(headstring));
if (!string.IsNullOrWhiteSpace(html))
{
try
{
if (headstring.ToLower().Contains("application/json"))
{
jsonViewResponse.Json = html;
}
}
catch
{
}
}
textview_responce.SetText(headstring + html);
this.webBrowser1.DocumentText = html;
}
}
// }
}
}
}
catch (Exception ex)
{
MessageBox.Show(string.Format("[异常] {0}", ex.ToString()));
debugmsg(string.Format("[异常] {0}", ex.ToString()));
}
//this.dataGridView1.Enabled = true;
}
private static void CompressBytesToStream(Stream stream)
{
using var compressor = new DeflateStream(stream, CompressionMode.Compress, leaveOpen: true);
compressor.Write(s_messageBytes, 0, s_messageBytes.Length);
}
public void CheckGetLengthProp()
{
MemoryStream backing = new MemoryStream(compressed_data);
DeflateStream decompressing = new DeflateStream(backing, CompressionMode.Decompress);
long length = decompressing.Length;
}
public async Task <ServiceResult> SendRequestWithBodyAsync(string url, byte[] data)
{
ServiceResult result = new ServiceResult();
try
{
HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create(url);
// настройка запроса
this.ConfigureRequest(ref httpWebRequest);
httpWebRequest.Method = WebRequestMethods.Http.Post;
// содержимое запроса
httpWebRequest.ContentLength = data.Length;
using (Stream stream = httpWebRequest.GetRequestStream())
{
stream.Write(data, 0, data.Length);
}
using (HttpWebResponse response = (HttpWebResponse)await httpWebRequest.GetResponseAsync())
using (Stream responseStream = response.GetResponseStream())
{
Stream streamToRead = responseStream;
if (response.ContentEncoding.ToLower().Contains("gzip"))
{
streamToRead = new GZipStream(streamToRead, CompressionMode.Decompress);
}
else if (response.ContentEncoding.ToLower().Contains("deflate"))
{
streamToRead = new DeflateStream(streamToRead, CompressionMode.Decompress);
}
using (StreamReader streamReader = new StreamReader(streamToRead, Encoding.UTF8))
{
string answer = streamReader.ReadToEnd();
result.SetData(answer, (int)response.StatusCode);
}
}
}
#region EXCEPTIONS
catch (System.Net.WebException we)
{
System.Net.WebResponse response = we.Response;
string message = null;
if (response != null)
{
using (var stream = we.Response.GetResponseStream())
using (var reader = new System.IO.StreamReader(stream))
{
message = reader.ReadToEnd();
}
}
result.SetError(we, (int)we.Status, message);
return(result);
}
catch (Exception ex)
{
result.SetError(ex, EMPTY_STATUS_CODE);
return(result);
}
#endregion
return(result);
}
public static string Send(string url, string method, string data, HttpConfig config)
{
if (config == null)
{
config = new HttpConfig();
}
string result;
using (HttpWebResponse response = GetResponse(url, method, data, config))
{
Stream stream = response.GetResponseStream();
if (!String.IsNullOrEmpty(response.ContentEncoding))
{
if (response.ContentEncoding.Contains("gzip"))
{
stream = new GZipStream(stream, CompressionMode.Decompress);
}
else if (response.ContentEncoding.Contains("deflate"))
{
stream = new DeflateStream(stream, CompressionMode.Decompress);
}
}
byte[] bytes = null;
using (MemoryStream ms = new MemoryStream())
{
int count;
byte[] buffer = new byte[4096];
while ((count = stream.Read(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, count);
}
bytes = ms.ToArray();
}
#region 检测流编码
Encoding encoding;
//检测响应头是否返回了编码类型,若返回了编码类型则使用返回的编码
//注：有时响应头没有编码类型，CharacterSet经常设置为ISO-8859-1
if (!string.IsNullOrEmpty(response.CharacterSet) && response.CharacterSet.ToUpper() != "ISO-8859-1")
{
encoding = Encoding.GetEncoding(response.CharacterSet == "utf8" ? "utf-8" : response.CharacterSet);
}
else
{
//若没有在响应头找到编码，则去html找meta头的charset
result = Encoding.Default.GetString(bytes);
//在返回的html里使用正则匹配页面编码
Match match = Regex.Match(result, @"<meta.*charset=""?([\w-]+)""?.*>", RegexOptions.IgnoreCase);
if (match.Success)
{
encoding = Encoding.GetEncoding(match.Groups[1].Value);
}
else
{
//若html里面也找不到编码，默认使用utf-8
encoding = Encoding.GetEncoding(config.CharacterSet);
}
}
#endregion
result = encoding.GetString(bytes);
}
return(result);
}
public void TestReadSqliteImageDatabase3()
{
var watch = new Stopwatch();
watch.Start();
string exeName = Assembly.GetExecutingAssembly().Location;
string fileName = Path.Combine(Path.GetDirectoryName(exeName), @"..\..\..\..\..\..\Data\mnist_png.zip");
string dbFileName = Path.Combine(Path.GetDirectoryName(fileName), "mnist_vector.db");
using var conn = new SQLiteConnection();
conn.ConnectionString = $"Data Source={dbFileName}";
conn.Open();
using var cmd = conn.CreateCommand();
cmd.CommandText = "select [Group], Category, FileName, ImageVector from Images";
var categories = new Dictionary <string, int>();
using var reader = cmd.ExecuteReader();
while (reader.Read())
{
string group = Convert.ToString(reader.GetValue(0));
string category = Convert.ToString(reader.GetValue(1));
string imageFileName = Convert.ToString(reader.GetValue(2));
string groupCategory = $"{group}/{category}";
if (!categories.ContainsKey(groupCategory))
{
categories.Add(groupCategory, 1);
}
else
{
categories[groupCategory]++;
}
using var streamImage = reader.GetStream(3);
using var decompStream = new MemoryStream();
using var zipStream = new DeflateStream(streamImage, CompressionMode.Decompress);
zipStream.CopyTo(decompStream);
decompStream.Seek(0, SeekOrigin.Begin);
double[] dVector = new double[decompStream.Length / sizeof(double)];
unsafe
{
fixed(double *ptr = dVector)
{
long streamSize = dVector.Length * sizeof(double);
using var memVectorStream = new UnmanagedMemoryStream((byte *)ptr, streamSize, streamSize, FileAccess.Write);
decompStream.CopyTo(memVectorStream);
}
}
var vector = Vector <double> .Build.Dense(dVector);
}
watch.Stop();
output.WriteLine($"Reader SQLite db: {watch.ElapsedMilliseconds:N0} msec.");
output.WriteLine(string.Empty);
output.WriteLine("Categories:");
output.WriteLine("-----------");
foreach (KeyValuePair <string, int> pairCategory in categories)
{
output.WriteLine($@"{pairCategory.Key,-15}: {pairCategory.Value,8:N0}");
}
}
public void Constructor_InvalidCompressionMode()
{
DeflateStream ds = new DeflateStream(new MemoryStream(), (CompressionMode)Int32.MinValue);
}
/// <summary>
/// Compresses the specified buffer stream using the G-Zip compression algorithm.
/// </summary>
/// <param name="buffer">The buffer.</param>
/// <param name="method">The method.</param>
/// <param name="mode">The mode.</param>
/// <returns>Block of bytes of compressed stream</returns>
public static MemoryStream Compress(
this Stream buffer,
CompressionMethod method = CompressionMethod.Gzip,
CompressionMode mode = CompressionMode.Compress)
{
buffer.Position = 0;
var targetStream = new MemoryStream();
switch (method)
{
case CompressionMethod.Deflate:
if (mode == CompressionMode.Compress)
{
using (var compressor = new DeflateStream(targetStream, CompressionMode.Compress, true))
{
buffer.CopyTo(compressor, 1024);
buffer.CopyTo(compressor);
// WebSocket use this
targetStream.Write(LastByte, 0, 1);
targetStream.Position = 0;
}
}
else
{
using (var compressor = new DeflateStream(buffer, CompressionMode.Decompress))
{
compressor.CopyTo(targetStream);
}
}
break;
case CompressionMethod.Gzip:
if (mode == CompressionMode.Compress)
{
using (var compressor = new GZipStream(targetStream, CompressionMode.Compress, true))
{
buffer.CopyTo(compressor);
}
}
else
{
using (var compressor = new GZipStream(buffer, CompressionMode.Decompress))
{
compressor.CopyTo(targetStream);
}
}
break;
case CompressionMethod.None:
buffer.CopyTo(targetStream);
break;
default:
throw new ArgumentOutOfRangeException(nameof(method), method, null);
}
return(targetStream);
}
public async Task <ServiceResult> SendRequestAsync(string url)
{
ServiceResult result = new ServiceResult();
try
{
HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create(url);
// настройка запроса
this.ConfigureRequest(ref httpWebRequest);
httpWebRequest.Method = WebRequestMethods.Http.Get;
using (HttpWebResponse response = (HttpWebResponse)await httpWebRequest.GetResponseAsync())
using (Stream responseStream = response.GetResponseStream())
{
Stream streamToRead = responseStream;
if (response.ContentEncoding.ToLower().Contains("gzip"))
{
streamToRead = new GZipStream(streamToRead, CompressionMode.Decompress);
}
else if (response.ContentEncoding.ToLower().Contains("deflate"))
{
streamToRead = new DeflateStream(streamToRead, CompressionMode.Decompress);
}
else if (response.ContentType == "image/png")
{
MemoryStream ms = new MemoryStream();
streamToRead.CopyTo(ms);
byte[] buffer = ms.ToArray();
result.SetData(buffer, (int)response.StatusCode);
return(result);
}
using (StreamReader streamReader = new StreamReader(streamToRead, Encoding.UTF8))
{
string answer = streamReader.ReadToEnd();
result.SetData(answer, (int)response.StatusCode);
}
}
}
#region EXCEPTIONS
catch (System.Net.WebException we)
{
System.Net.WebResponse response = we.Response;
string message = null;
if (response != null)
{
using (var stream = we.Response.GetResponseStream())
using (var reader = new System.IO.StreamReader(stream))
{
message = reader.ReadToEnd();
}
}
result.SetError(we, (int)we.Status, message);
return(result);
}
catch (Exception ex)
{
result.SetError(ex, EMPTY_STATUS_CODE);
return(result);
}
#endregion
return(result);
}
/// <exception cref="PngProcessingException"/>
/// <exception cref="System.IO.IOException"/>
private static IEnumerable <Directory> ProcessChunk(PngChunk chunk)
{
var chunkType = chunk.ChunkType;
var bytes = chunk.Bytes;
if (chunkType == PngChunkType.IHDR)
{
var header = new PngHeader(bytes);
var directory = new PngDirectory(PngChunkType.IHDR);
directory.Set(PngDirectory.TagImageWidth, header.ImageWidth);
directory.Set(PngDirectory.TagImageHeight, header.ImageHeight);
directory.Set(PngDirectory.TagBitsPerSample, header.BitsPerSample);
directory.Set(PngDirectory.TagColorType, header.ColorType.NumericValue);
directory.Set(PngDirectory.TagCompressionType, header.CompressionType);
directory.Set(PngDirectory.TagFilterMethod, header.FilterMethod);
directory.Set(PngDirectory.TagInterlaceMethod, header.InterlaceMethod);
yield return(directory);
}
else if (chunkType == PngChunkType.PLTE)
{
var directory = new PngDirectory(PngChunkType.PLTE);
directory.Set(PngDirectory.TagPaletteSize, bytes.Length / 3);
yield return(directory);
}
else if (chunkType == PngChunkType.tRNS)
{
var directory = new PngDirectory(PngChunkType.tRNS);
directory.Set(PngDirectory.TagPaletteHasTransparency, 1);
yield return(directory);
}
else if (chunkType == PngChunkType.sRGB)
{
int srgbRenderingIntent = unchecked ((sbyte)bytes[0]);
var directory = new PngDirectory(PngChunkType.sRGB);
directory.Set(PngDirectory.TagSrgbRenderingIntent, srgbRenderingIntent);
yield return(directory);
}
else if (chunkType == PngChunkType.cHRM)
{
var chromaticities = new PngChromaticities(bytes);
var directory = new PngChromaticitiesDirectory();
directory.Set(PngChromaticitiesDirectory.TagWhitePointX, chromaticities.WhitePointX);
directory.Set(PngChromaticitiesDirectory.TagWhitePointY, chromaticities.WhitePointY);
directory.Set(PngChromaticitiesDirectory.TagRedX, chromaticities.RedX);
directory.Set(PngChromaticitiesDirectory.TagRedY, chromaticities.RedY);
directory.Set(PngChromaticitiesDirectory.TagGreenX, chromaticities.GreenX);
directory.Set(PngChromaticitiesDirectory.TagGreenY, chromaticities.GreenY);
directory.Set(PngChromaticitiesDirectory.TagBlueX, chromaticities.BlueX);
directory.Set(PngChromaticitiesDirectory.TagBlueY, chromaticities.BlueY);
yield return(directory);
}
else if (chunkType == PngChunkType.gAMA)
{
var gammaInt = ByteConvert.ToInt32BigEndian(bytes);
var directory = new PngDirectory(PngChunkType.gAMA);
directory.Set(PngDirectory.TagGamma, gammaInt / 100000.0);
yield return(directory);
}
else if (chunkType == PngChunkType.iCCP)
{
var reader = new SequentialByteArrayReader(bytes);
var profileName = reader.GetNullTerminatedStringValue(maxLengthBytes: 79);
var directory = new PngDirectory(PngChunkType.iCCP);
directory.Set(PngDirectory.TagIccProfileName, profileName);
var compressionMethod = reader.GetSByte();
if (compressionMethod == 0)
{
// Only compression method allowed by the spec is zero: deflate
// This assumes 1-byte-per-char, which it is by spec.
var bytesLeft = bytes.Length - profileName.Bytes.Length - 2;
// http://george.chiramattel.com/blog/2007/09/deflatestream-block-length-does-not-match.html
// First two bytes are part of the zlib specification (RFC 1950), not the deflate specification (RFC 1951).
reader.Skip(2);
bytesLeft -= 2;
var compressedProfile = reader.GetBytes(bytesLeft);
IccDirectory?iccDirectory = null;
Exception? ex = null;
try
{
using var inflaterStream = new DeflateStream(new MemoryStream(compressedProfile), CompressionMode.Decompress);
iccDirectory = new IccReader().Extract(new IndexedCapturingReader(inflaterStream));
iccDirectory.Parent = directory;
}
catch (Exception e)
{
ex = e;
}
if (iccDirectory != null)
{
yield return(iccDirectory);
}
else if (ex != null)
{
directory.AddError($"Exception decompressing {nameof(PngChunkType.iCCP)} chunk: {ex.Message}");
}
}
else
{
directory.AddError("Invalid compression method value");
}
yield return(directory);
}
else if (chunkType == PngChunkType.bKGD)
{
var directory = new PngDirectory(PngChunkType.bKGD);
directory.Set(PngDirectory.TagBackgroundColor, bytes);
yield return(directory);
}
else if (chunkType == PngChunkType.tEXt)
{
var reader = new SequentialByteArrayReader(bytes);
var keyword = reader.GetNullTerminatedStringValue(maxLengthBytes: 79).ToString(_latin1Encoding);
var bytesLeft = bytes.Length - keyword.Length - 1;
var value = reader.GetNullTerminatedStringValue(bytesLeft, _latin1Encoding);
var textPairs = new List <KeyValuePair> {
new KeyValuePair(keyword, value)
};
var directory = new PngDirectory(PngChunkType.tEXt);
directory.Set(PngDirectory.TagTextualData, textPairs);
yield return(directory);
}
else if (chunkType == PngChunkType.zTXt)
{
var reader = new SequentialByteArrayReader(bytes);
var keyword = reader.GetNullTerminatedStringValue(maxLengthBytes: 79).ToString(_latin1Encoding);
var compressionMethod = reader.GetSByte();
var bytesLeft = bytes.Length - keyword.Length - 1 - 1 - 1 - 1;
byte[]? textBytes = null;
if (compressionMethod == 0)
{
if (!TryDeflate(bytes, bytesLeft, out textBytes, out string?errorMessage))
{
var directory = new PngDirectory(PngChunkType.zTXt);
directory.AddError($"Exception decompressing {nameof(PngChunkType.zTXt)} chunk with keyword \"{keyword}\": {errorMessage}");
yield return(directory);
}
}
else
{
var directory = new PngDirectory(PngChunkType.zTXt);
directory.AddError("Invalid compression method value");
yield return(directory);
}
if (textBytes != null)
{
foreach (var directory in ProcessTextChunk(keyword, textBytes))
{
yield return(directory);
}
}
}
else if (chunkType == PngChunkType.iTXt)
{
var reader = new SequentialByteArrayReader(bytes);
var keyword = reader.GetNullTerminatedStringValue(maxLengthBytes: 79).ToString(_latin1Encoding);
var compressionFlag = reader.GetSByte();
var compressionMethod = reader.GetSByte();
// TODO we currently ignore languageTagBytes and translatedKeywordBytes
var languageTagBytes = reader.GetNullTerminatedBytes(bytes.Length);
var translatedKeywordBytes = reader.GetNullTerminatedBytes(bytes.Length);
var bytesLeft = bytes.Length - keyword.Length - 1 - 1 - 1 - languageTagBytes.Length - 1 - translatedKeywordBytes.Length - 1;
byte[]? textBytes = null;
if (compressionFlag == 0)
{
textBytes = reader.GetNullTerminatedBytes(bytesLeft);
}
else if (compressionFlag == 1)
{
if (compressionMethod == 0)
{
if (!TryDeflate(bytes, bytesLeft, out textBytes, out string?errorMessage))
{
var directory = new PngDirectory(PngChunkType.iTXt);
directory.AddError($"Exception decompressing {nameof(PngChunkType.iTXt)} chunk with keyword \"{keyword}\": {errorMessage}");
yield return(directory);
}
}
else
{
var directory = new PngDirectory(PngChunkType.iTXt);
directory.AddError("Invalid compression method value");
yield return(directory);
}
}
else
{
var directory = new PngDirectory(PngChunkType.iTXt);
directory.AddError("Invalid compression flag value");
yield return(directory);
}
if (textBytes != null)
{
foreach (var directory in ProcessTextChunk(keyword, textBytes))
{
yield return(directory);
}
}
}
else if (chunkType == PngChunkType.tIME)
{
var reader = new SequentialByteArrayReader(bytes);
var year = reader.GetUInt16();
var month = reader.GetByte();
int day = reader.GetByte();
int hour = reader.GetByte();
int minute = reader.GetByte();
int second = reader.GetByte();
var directory = new PngDirectory(PngChunkType.tIME);
if (DateUtil.IsValidDate(year, month, day) && DateUtil.IsValidTime(hour, minute, second))
{
var time = new DateTime(year, month, day, hour, minute, second, DateTimeKind.Unspecified);
directory.Set(PngDirectory.TagLastModificationTime, time);
}
else
{
directory.AddError($"PNG tIME data describes an invalid date/time: year={year} month={month} day={day} hour={hour} minute={minute} second={second}");
}
yield return(directory);
}
else if (chunkType == PngChunkType.pHYs)
{
var reader = new SequentialByteArrayReader(bytes);
var pixelsPerUnitX = reader.GetInt32();
var pixelsPerUnitY = reader.GetInt32();
var unitSpecifier = reader.GetSByte();
var directory = new PngDirectory(PngChunkType.pHYs);
directory.Set(PngDirectory.TagPixelsPerUnitX, pixelsPerUnitX);
directory.Set(PngDirectory.TagPixelsPerUnitY, pixelsPerUnitY);
directory.Set(PngDirectory.TagUnitSpecifier, unitSpecifier);
yield return(directory);
}
else if (chunkType.Equals(PngChunkType.sBIT))
{
var directory = new PngDirectory(PngChunkType.sBIT);
directory.Set(PngDirectory.TagSignificantBits, bytes);
yield return(directory);
}
yield break;
IEnumerable <Directory> ProcessTextChunk(string keyword, byte[] textBytes)
{
if (keyword == "XML:com.adobe.xmp")
{
yield return(new XmpReader().Extract(textBytes));
}
else if (keyword == "Raw profile type xmp")
{
if (TryProcessRawProfile(out int byteCount))
{
yield return(new XmpReader().Extract(textBytes, 0, byteCount));
}
else
{
yield return(ReadTextDirectory(keyword, textBytes, chunkType));
}
}
else if (keyword == "Raw profile type exif" || keyword == "Raw profile type APP1")
{
if (TryProcessRawProfile(out _))
{
foreach (var exifDirectory in new ExifReader().Extract(new ByteArrayReader(textBytes)))
{
yield return(exifDirectory);
}
}
else
{
yield return(ReadTextDirectory(keyword, textBytes, chunkType));
}
}
else if (keyword == "Raw profile type icc" || keyword == "Raw profile type icm")
{
if (TryProcessRawProfile(out _))
{
yield return(new IccReader().Extract(new ByteArrayReader(textBytes)));
}
else
{
yield return(ReadTextDirectory(keyword, textBytes, chunkType));
}
}
else if (keyword == "Raw profile type iptc")
{
if (TryProcessRawProfile(out int byteCount))
{
yield return(new IptcReader().Extract(new SequentialByteArrayReader(textBytes), byteCount));
}
else
{
yield return(ReadTextDirectory(keyword, textBytes, chunkType));
}
}
else
{
yield return(ReadTextDirectory(keyword, textBytes, chunkType));
}
PngDirectory ReadTextDirectory(string keyword, byte[] textBytes, PngChunkType pngChunkType)
{
var textPairs = new[] { new KeyValuePair(keyword, new StringValue(textBytes, _latin1Encoding)) };
var directory = new PngDirectory(pngChunkType);
directory.Set(PngDirectory.TagTextualData, textPairs);
return(directory);
}
bool TryProcessRawProfile(out int byteCount)
{
// Raw profiles have form "\n<name>\n<length>\n<hex>\n"
if (textBytes.Length == 0 || textBytes[0] != '\n')
{
byteCount = default;
return(false);
}
var i = 1;
// Skip name
while (i < textBytes.Length && textBytes[i] != '\n')
{
i++;
}
if (i == textBytes.Length)
{
byteCount = default;
return(false);
}
// Read length
int length = 0;
while (true)
{
i++;
var c = (char)textBytes[i];
if (c == ' ')
{
continue;
}
if (c == '\n')
{
break;
}
if (c >= '0' && c <= '9')
{
length *= 10;
length += c - '0';
}
else
{
byteCount = default;
return(false);
}
}
i++;
// We should be at the ASCII-encoded hex data. Walk through the remaining bytes, re-writing as raw bytes
// starting at offset zero in the array. We have to skip \n characters.
// Validate the data can be correctly parsed before modifying it in-place, because if parsing fails later
// consumers may want the unmodified data.
// Each row must have 72 characters (36 bytes once decoded) separated by \n
const int rowCharCount = 72;
int charsInRow = rowCharCount;
for (int j = i; j < length + i; j++)
{
byte c = textBytes[j];
if (charsInRow-- == 0)
{
if (c != '\n')
{
byteCount = default;
return(false);
}
charsInRow = rowCharCount;
continue;
}
if ((c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F'))
{
byteCount = default;
return(false);
}
}
byteCount = length;
var writeIndex = 0;
charsInRow = rowCharCount;
while (length > 0)
{
var c1 = textBytes[i++];
if (charsInRow-- == 0)
{
Debug.Assert(c1 == '\n');
charsInRow = rowCharCount;
continue;
}
var c2 = textBytes[i++];
charsInRow--;
var n1 = ParseHexNibble(c1);
var n2 = ParseHexNibble(c2);
length--;
textBytes[writeIndex++] = (byte)((n1 << 4) | n2);
}
return(writeIndex == byteCount);
static int ParseHexNibble(int h)
{
if (h >= '0' && h <= '9')
{
return(h - '0');
}
if (h >= 'a' && h <= 'f')
{
return(10 + (h - 'a'));
}
if (h >= 'A' && h <= 'F')
{
return(10 + (h - 'A'));
}
Debug.Fail("Should not reach here");
throw new InvalidOperationException();
}
}
public void TestSqliteImagesDatabase2()
{
var watch = new Stopwatch();
watch.Start();
string exeName = Assembly.GetExecutingAssembly().Location;
string fileName = Path.Combine(Path.GetDirectoryName(exeName), @"..\..\..\..\..\..\Data\mnist_png.zip");
string dbFileName = Path.Combine(Path.GetDirectoryName(fileName), "mnist_vector.db");
if (File.Exists(dbFileName))
{
File.Delete(dbFileName);
}
SQLiteConnection.CreateFile(dbFileName);
using var conn = new SQLiteConnection();
conn.ConnectionString = $"Data Source={dbFileName}";
conn.Open();
using var session = conn.BeginTransaction();
try
{
using var cmdCreateTable = conn.CreateCommand();
cmdCreateTable.CommandText = "create table Images ([Group] text, Category text, FileName text, ImageVector blob)";
cmdCreateTable.ExecuteNonQuery();
using var cmdInsert = conn.CreateCommand();
cmdInsert.CommandText = "insert into Images([Group], Category, FileName, ImageVector) values (@group, @category, @fileName, @imageVector)";
var paramGroup = cmdInsert.Parameters.Add("@group", System.Data.DbType.String);
var paramCategory = cmdInsert.Parameters.Add("@category", System.Data.DbType.String);
var paramFileName = cmdInsert.Parameters.Add("@fileName", System.Data.DbType.String);
var paramImageVector = cmdInsert.Parameters.Add("@imageVector", System.Data.DbType.Binary);
cmdInsert.Prepare();
using var zip = ZipFile.OpenRead(fileName);
foreach (var entry in zip.Entries)
{
if (string.Compare(Path.GetExtension(entry.FullName), ".png", true) != 0)
{
continue;
}
using var stream = entry.Open();
var vector = BitmapHelper.BitmapToVector(stream);
var dVector = vector.AsArray() ?? vector.ToArray();
using var compStream = new MemoryStream();
using var zipStream = new DeflateStream(compStream, CompressionLevel.Optimal);
unsafe
{
fixed(double *ptr = dVector)
{
long streamSize = dVector.Length * sizeof(double);
using var memVectorStream = new UnmanagedMemoryStream((byte *)ptr, streamSize, streamSize, FileAccess.Read);
memVectorStream.CopyTo(zipStream);
}
}
zipStream.Flush();
compStream.Seek(0, SeekOrigin.Begin);
var bCompVector = compStream.ToArray();
var folder = Path.GetDirectoryName(entry.FullName);
string group, category;
var matchFolder = Regex.Match(folder, @"[\\/]\s*(?<Group>.*?)\s*[\\/]\s*(?<Category>.*?)\s*$");
if (matchFolder.Success)
{
group = matchFolder.Groups["Group"].Value;
category = matchFolder.Groups["Category"].Value;
}
else
{
group = "unknown";
category = "unknown";
}
paramGroup.Value = group;
paramCategory.Value = category;
paramFileName.Value = entry.Name;
paramImageVector.Value = bCompVector;
cmdInsert.ExecuteNonQuery();
}
}
finally
{
session.Commit();
}
watch.Stop();
output.WriteLine($"Write SQLite db: {watch.ElapsedMilliseconds:N0} msec.");
}
public static Bitmap Load(Stream s)
{
using (var br = new BinaryReader(s))
{
var signature = new byte[] { 137, 80, 78, 71, 13, 10, 26, 10 };
foreach (var b in signature)
{
if (br.ReadByte() != b)
{
throw new InvalidDataException("PNG Signature is bogus");
}
}
Bitmap bitmap = null;
Color[] palette = null;
List <byte> data = new List <byte>();
for (; ;)
{
var length = IPAddress.NetworkToHostOrder(br.ReadInt32());
var type = Encoding.UTF8.GetString(br.ReadBytes(4));
var content = br.ReadBytes(length);
/*var crc = */ br.ReadInt32();
using (var ms = new MemoryStream(content))
using (var cr = new BinaryReader(ms))
switch (type)
{
case "IHDR":
{
var width = IPAddress.NetworkToHostOrder(cr.ReadInt32());
var height = IPAddress.NetworkToHostOrder(cr.ReadInt32());
var bitDepth = cr.ReadByte();
var colorType = (PngColorType)cr.ReadByte();
var compression = cr.ReadByte();
/*var filter = */ cr.ReadByte();
var interlace = cr.ReadByte();
if (compression != 0)
{
throw new InvalidDataException("Compression method not supported");
}
if (interlace != 0)
{
throw new InvalidDataException("Interlacing not supported");
}
bitmap = new Bitmap(width, height, MakePixelFormat(bitDepth, colorType));
}
break;
case "PLTE":
{
palette = new Color[256];
for (var i = 0; i < 256; i++)
{
var r = cr.ReadByte(); var g = cr.ReadByte(); var b = cr.ReadByte();
palette[i] = Color.FromArgb(r, g, b);
}
}
break;
case "tRNS":
{
for (var i = 0; i < length; i++)
{
palette[i] = Color.FromArgb(cr.ReadByte(), palette[i]);
}
}
break;
case "IDAT":
{
data.AddRange(content);
}
break;
case "IEND":
{
var bits = bitmap.LockBits(new Rectangle(0, 0, bitmap.Width, bitmap.Height),
ImageLockMode.WriteOnly, PixelFormat.Format8bppIndexed);
using (var ns = new MemoryStream(data.ToArray()))
{
// 'zlib' flags bytes; confuses the DeflateStream.
/*var flags = (byte)*/ ns.ReadByte();
/*var moreFlags = (byte)*/ ns.ReadByte();
using (var ds = new DeflateStream(ns, CompressionMode.Decompress))
using (var dr = new BinaryReader(ds))
{
var prevLine = new byte[bitmap.Width]; // all zero
for (var y = 0; y < bitmap.Height; y++)
{
var filter = (PngFilter)dr.ReadByte();
var line = dr.ReadBytes(bitmap.Width);
for (var i = 0; i < bitmap.Width; i++)
{
line[i] = i > 0
? UnapplyFilter(filter, line[i], line[i - 1], prevLine[i], prevLine[i - 1])
: UnapplyFilter(filter, line[i], 0, prevLine[i], 0);
}
Marshal.Copy(line, 0, new IntPtr(bits.Scan0.ToInt64() + y * bits.Stride), line.Length);
prevLine = line;
}
}
}
bitmap.UnlockBits(bits);
using (var temp = new Bitmap(1, 1, PixelFormat.Format8bppIndexed))
{
var cp = temp.Palette;
for (var i = 0; i < 256; i++)
{
cp.Entries[i] = palette[i]; // finalize the palette.
}
bitmap.Palette = cp;
return(bitmap);
}
}
}
}
}
}
public string Open(string uri, string postData, string referer)
{
HttpWebRequest _request;
int i = 0;
var sw = new Stopwatch();
sw.Start();
//WaitAMoment();
//lock (_lockOpenSerialize)
{
string html = null;
HttpWebResponse response = null;
Stream responseStream = null;
StreamReader sr = null;
try
{
if (RequestInterval > 0)
{
TimeSpan ts = new TimeSpan(DateTime.Now.Ticks - _ticks);
double sleep = ts.TotalMilliseconds;
if (sleep < RequestInterval)
{
Thread.Sleep((int)(RequestInterval - sleep));
}
_ticks = DateTime.Now.Ticks;
}
if (referer == null)
{
referer = _referer;
}
if (BeginRequest != null)
{
HttpClientEventArgs e = new HttpClientEventArgs();
e.Uri = uri;
e.PostData = postData;
e.Referer = referer;
BeginRequest(e);
}
if (uri.StartsWith("https", StringComparison.OrdinalIgnoreCase))
{
ServicePointManager.ServerCertificateValidationCallback = CheckValidationResult;
_request = (HttpWebRequest)WebRequest.Create(uri);
_request.ProtocolVersion = HttpVersion.Version10;
}
else
{
_request = (HttpWebRequest)WebRequest.Create(uri);
}
_request.Referer = referer;
_request.Headers.Add("x-requested-with", "XMLHttpRequest");
_request.Accept = _accept;
_request.UserAgent = _userAgent;
_request.CookieContainer = _cookieContainer;
_request.Timeout = Timeout;
if (_proxy != null)
{
_request.Proxy = _proxy;
}
_request.AllowWriteStreamBuffering = true;
_request.AllowAutoRedirect = AllowAutoRedirect;
_request.KeepAlive = true;
if (AddRequestHeader != null)
{
foreach (var addheader in AddRequestHeader)
{
_request.Headers.Add(addheader.Key, addheader.Value);
}
}
foreach (Cookie cookie in _cookieCollection)
{
_cookieContainer.SetCookies(_request.RequestUri, cookie.Name + "=" + cookie.Value);
}
if (string.IsNullOrEmpty(postData))
{
_request.Method = "GET";
}
else
{
byte[] bytes = Encoding.GetEncoding(_charset).GetBytes(postData);
_request.Method = "POST";
_request.ContentType = _contentType;
_request.ContentLength = bytes.Length;
using (Stream writer = _request.GetRequestStream())
{
writer.Write(bytes, 0, bytes.Length);
}
}
response = (HttpWebResponse)_request.GetResponse();
ResponeHeader = response.Headers;
#region Cookie管理
// 1.当前地址的cookie （基础类自动处理，不会处理非当前路径的cookie）
_cookieContainer = _request.CookieContainer;
// 2. 处理response返回的所有cookie(非当前url路径的)
string allCookies = response.Headers["Set-Cookie"];
if (!string.IsNullOrWhiteSpace(allCookies))
{
string pathSplit = "path=";
while (allCookies.Contains(pathSplit))
{
#region 分解["Set-Cookie"]字符串
string cookieText =
allCookies.Substring(0, allCookies.IndexOf(pathSplit,
StringComparison.CurrentCultureIgnoreCase));
allCookies = allCookies.Substring($"{cookieText}{pathSplit}".Length);
int nextCookieSplitIndex = allCookies.IndexOf(",",
StringComparison.CurrentCultureIgnoreCase);
string cookiePath = nextCookieSplitIndex == -1
? allCookies
: allCookies.Substring(0, nextCookieSplitIndex);
allCookies = nextCookieSplitIndex == -1
? string.Empty
: allCookies.Substring($"{cookiePath},".Length);
#endregion
#region 注入 _cookieContainer
string cookiePathFull;
cookieText = cookieText.Trim();
cookiePath = cookiePath.Trim();
if (cookiePath.Contains("://"))
{
cookiePathFull = cookiePath;
}
else
{
cookiePathFull = $"{_request.RequestUri.Scheme}://{_request.RequestUri.Authority}{cookiePath}";
}
_cookieContainer.SetCookies(new Uri(cookiePathFull), cookieText);
#endregion
}
}
#endregion
if (response.ContentEncoding.ToLower().Contains("gzip"))
{
responseStream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress);
}
else if (response.ContentEncoding.ToLower().Contains("deflate"))
{
responseStream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress);
}
else
{
responseStream = response.GetResponseStream();
}
sr = new StreamReader(responseStream, Encoding.GetEncoding(_charset));
html = sr.ReadToEnd();
if (EndRequest != null)
{
HttpClientEventArgs e = new HttpClientEventArgs();
e.Uri = uri;
e.PostData = postData;
e.Referer = referer;
e.ResponseText = html;
EndRequest(e);
}
}
catch (Exception ex)
{
//LogCache.Instance.AddInfo(
// $"HttpClient.Open() 发生异常。(url:{uri},postdata:{postData},referer{referer}) \n异常详情：{ex}\n{ex.StackTrace}");
//if (Error != null)
//{
// Error(ex);
//}
log.Error("请求发生异常", ex);
}
finally
{
_referer = uri;
if (response != null)
{
response.Close();
response = null;
}
if (responseStream != null)
{
responseStream.Close();
responseStream = null;
}
if (sr != null)
{
sr.Close();
sr = null;
}
if (html == null)
{
log.Error($"请求html为null,url:{uri},postdata:{postData}");
//LogCache.Instance.AddInfo(
// $"HttpClient.Open() 请求html为null。(invokmethod{new StackFrame(1).GetMethod()}, url:{uri},postdata:{postData},referer{referer}) ");
}
}
sw.Stop();
//log.Debug($"{i}请求服务花费了{sw.ElapsedMilliseconds}ms。");
i++;
return(html);
}
}
/// <summary>
/// Decompresses the byte array. The byte array must have been previously compressed with the Compress() method.
/// </summary>
/// <param name="algorithm">The compression algorithm to use when decompressing. This must be the same compression algorithm used to compress the data. Valid values are defined as constants in the CompressionAlgorithm class. The default is to use zlib format.</param>
public void Uncompress(string algorithm)
{
ValidationUtils.ArgumentConditionTrue(algorithm == CompressionAlgorithm.Deflate || algorithm == CompressionAlgorithm.Zlib, "algorithm", "Invalid parameter");
#if SILVERLIGHT
throw new NotSupportedException();
#else
if (algorithm == CompressionAlgorithm.Zlib)
{
//The zlib format is specified by RFC 1950. Zlib also uses deflate, plus 2 or 6 header bytes, and a 4 byte checksum at the end.
//The first 2 bytes indicate the compression method and flags. If the dictionary flag is set, then 4 additional bytes will follow.
//Preset dictionaries aren't very common and we don't support them
Position = 0;
ZlibStream deflateStream = new ZlibStream(_memoryStream, CompressionMode.Decompress, false);
MemoryStream ms = new MemoryStream();
byte[] buffer = new byte[1024];
// Chop off the first two bytes
//int b = _memoryStream.ReadByte();
//b = _memoryStream.ReadByte();
while (true)
{
int readCount = deflateStream.Read(buffer, 0, buffer.Length);
if (readCount > 0)
{
ms.Write(buffer, 0, readCount);
}
else
{
break;
}
}
deflateStream.Close();
_memoryStream.Close();
_memoryStream.Dispose();
_memoryStream = ms;
_memoryStream.Position = 0;
}
if (algorithm == CompressionAlgorithm.Deflate)
{
Position = 0;
DeflateStream deflateStream = new DeflateStream(_memoryStream, CompressionMode.Decompress, false);
MemoryStream ms = new MemoryStream();
byte[] buffer = new byte[1024];
while (true)
{
int readCount = deflateStream.Read(buffer, 0, buffer.Length);
if (readCount > 0)
{
ms.Write(buffer, 0, readCount);
}
else
{
break;
}
}
deflateStream.Close();
_memoryStream.Close();
_memoryStream.Dispose();
_memoryStream = ms;
_memoryStream.Position = 0;
}
AMFReader amfReader = new AMFReader(_memoryStream);
AMFWriter amfWriter = new AMFWriter(_memoryStream);
_dataOutput = new DataOutput(amfWriter);
_dataInput = new DataInput(amfReader);
#endif
}
public static bool ConnectIPC(string ip)
{
try
{
string pageSource = "";
HttpWebRequest request = (HttpWebRequest)WebRequest.Create("http://" + ip);
request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
request.AllowWriteStreamBuffering = false; //禁止缓冲加快载入速度
request.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate"); //定义gzip压缩页面支持
request.ContentType = "application/json; charset=UTF-8"; //定义文档类型及编码
request.AllowAutoRedirect = false; //禁止自动跳转
//设置User-Agent，伪装成Google Chrome浏览器
request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36";
request.Timeout = 9000; //定义请求超时时间为5秒
request.KeepAlive = false; //启用长连接
request.Method = "GET"; //定义请求方式为GET
using (var response = (HttpWebResponse)request.GetResponse())
{
if (response.ContentEncoding.ToLower().Contains("gzip"))//解压
{
using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
{
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{
pageSource = reader.ReadToEnd();
}
}
}
else if (response.ContentEncoding.ToLower().Contains("deflate"))//解压
{
using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress))
{
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{
pageSource = reader.ReadToEnd();
}
}
}
else
{
using (Stream stream = response.GetResponseStream())//原始
{
using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
{
pageSource = reader.ReadToEnd();
//File.WriteAllText(".\\test.txt", pageSource);
if (!string.IsNullOrEmpty(pageSource))
{
return(true);
}
}
}
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
return(false);
}
return(false);
}
async Task <HttpResponseMessage> DoProcessRequest(HttpRequestMessage request, URL javaUrl, HttpURLConnection httpConnection, CancellationToken cancellationToken)
{
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"{this}.DoProcessRequest ()");
}
try {
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $" connecting");
}
await httpConnection.ConnectAsync().ConfigureAwait(false);
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $" connected");
}
} catch (Java.Net.ConnectException ex) {
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Connection exception {ex}");
}
// Wrap it nicely in a "standard" exception so that it's compatible with HttpClientHandler
throw new WebException(ex.Message, ex, WebExceptionStatus.ConnectFailure, null);
}
if (httpConnection.DoOutput)
{
await request.Content.CopyToAsync(httpConnection.OutputStream).ConfigureAwait(false);
}
var statusCode = (HttpStatusCode)httpConnection.ResponseCode;
var connectionUri = new Uri(httpConnection.URL.ToString());
// If the request was redirected we need to put the new URL in the request
request.RequestUri = connectionUri;
var ret = new AndroidHttpResponseMessage(javaUrl, httpConnection)
{
RequestMessage = request,
ReasonPhrase = httpConnection.ResponseMessage,
StatusCode = statusCode,
};
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Status code: {statusCode}");
}
if (statusCode == HttpStatusCode.Unauthorized || statusCode == HttpStatusCode.ProxyAuthenticationRequired)
{
// We don't resend the request since that would require new set of credentials if the
// ones provided in Credentials are invalid (or null) and that, in turn, may require asking the
// user which is not something that should be taken care of by us and in this
// context. The application should be responsible for this.
// HttpClientHandler throws an exception in this instance, but I think it's not a good
// idea. We'll return the response message with all the information required by the
// application to fill in the blanks and provide the requested credentials instead.
//
// We should return the body of the response too but, alas, the Java client will throw
// a, wait for it, FileNotFound exception if we attempt to access the input stream. So
// no body, just a dummy. Java FTW!
ret.Content = new StringContent("Unauthorized", Encoding.ASCII);
CopyHeaders(httpConnection, ret);
if (ret.Headers.WwwAuthenticate != null)
{
ProxyAuthenticationRequested = false;
CollectAuthInfo(ret.Headers.WwwAuthenticate);
}
else if (ret.Headers.ProxyAuthenticate != null)
{
ProxyAuthenticationRequested = true;
CollectAuthInfo(ret.Headers.ProxyAuthenticate);
}
ret.RequestedAuthentication = RequestedAuthentication;
return(ret);
}
if (!IsErrorStatusCode(statusCode))
{
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Reading...");
}
Stream inputStream = new BufferedStream(httpConnection.InputStream);
if (decompress_here)
{
string[] encodings = httpConnection.ContentEncoding?.Split(',');
if (encodings != null)
{
if (encodings.Contains(GZIP_ENCODING, StringComparer.OrdinalIgnoreCase))
{
inputStream = new GZipStream(inputStream, CompressionMode.Decompress);
}
else if (encodings.Contains(DEFLATE_ENCODING, StringComparer.OrdinalIgnoreCase))
{
inputStream = new DeflateStream(inputStream, CompressionMode.Decompress);
}
}
}
ret.Content = new StreamContent(inputStream);
}
else
{
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Status code is {statusCode}, returning empty content");
}
// For 400 >= response code <= 599 the Java client throws the FileNotFound exeption when attempting to read from the connection
// Client tests require we return no content here
ret.Content = new StringContent(String.Empty, Encoding.ASCII);
}
CopyHeaders(httpConnection, ret);
IEnumerable <string> cookieHeaderValue;
if (!UseCookies || CookieContainer == null || !ret.Headers.TryGetValues("Set-Cookie", out cookieHeaderValue) || cookieHeaderValue == null)
{
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"No cookies");
}
return(ret);
}
try {
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Parsing cookies");
}
CookieContainer.SetCookies(connectionUri, String.Join(",", cookieHeaderValue));
} catch (Exception ex) {
// We don't want to terminate the response because of a bad cookie, hence just reporting
// the issue. We might consider adding a virtual method to let the user handle the
// issue, but not sure if it's really needed. Set-Cookie header will be part of the
// header collection so the user can always examine it if they spot an error.
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Failed to parse cookies in the server response. {ex.GetType ()}: {ex.Message}");
}
}
if (Logger.LogNet)
{
Logger.Log(LogLevel.Info, LOG_APP, $"Returning");
}
return(ret);
}
public abstract Task WriteAsync(DeflateStream unzip, byte[] buffer, int offset, int count);
public byte[] GetRawData(Compression compressMode)
{
var bytes = new List <byte>();
//Add header - signature ("QRR")
bytes.AddRange(new byte[] { 0x51, 0x52, 0x52, 0x00 });
//Add header - rowsize
bytes.Add((byte)ModuleMatrix.Count);
//Build data queue
var dataQueue = new Queue <int>();
foreach (var row in ModuleMatrix)
{
foreach (var module in row)
{
dataQueue.Enqueue((bool)module ? 1 : 0);
}
}
for (int i = 0; i < 8 - (ModuleMatrix.Count * ModuleMatrix.Count) % 8; i++)
{
dataQueue.Enqueue(0);
}
//Process queue
while (dataQueue.Count > 0)
{
byte b = 0;
for (int i = 7; i >= 0; i--)
{
b += (byte)(dataQueue.Dequeue() << i);
}
bytes.Add(b);
}
var rawData = bytes.ToArray();
//Compress stream (optional)
if (compressMode == Compression.Deflate)
{
using (var output = new MemoryStream())
{
using (var dstream = new DeflateStream(output, CompressionMode.Compress))
{
dstream.Write(rawData, 0, rawData.Length);
}
rawData = output.ToArray();
}
}
else if (compressMode == Compression.GZip)
{
using (var output = new MemoryStream())
{
using (GZipStream gzipStream = new GZipStream(output, CompressionMode.Compress, true))
{
gzipStream.Write(rawData, 0, rawData.Length);
}
rawData = output.ToArray();
}
}
return(rawData);
}
