public static byte[] Decompress(this Byte[] bytes)
{
using (MemoryStream tempMs = new MemoryStream()) {
using (MemoryStream ms = new MemoryStream(bytes)) {
GZipStream Decompress = new GZipStream(ms, CompressionMode.Decompress);
Decompress.CopyTo(tempMs);
Decompress.Close();
return tempMs.ToArray();
}
}
}
public void MutateIncoming(TransportMessage transportMessage)
{
if (!transportMessage.Headers.ContainsKey("IWasCompressed"))
{
return;
}
using (GZipStream bigStream = new GZipStream(new MemoryStream(transportMessage.Body), CompressionMode.Decompress))
{
MemoryStream bigStreamOut = new MemoryStream();
bigStream.CopyTo(bigStreamOut);
transportMessage.Body = bigStreamOut.ToArray();
}
}
public Task MutateIncoming(MutateIncomingTransportMessageContext context)
{
if (!context.Headers.ContainsKey("IWasCompressed"))
{
return Task.FromResult(0);
}
using (GZipStream bigStream = new GZipStream(new MemoryStream(context.Body), CompressionMode.Decompress))
{
MemoryStream bigStreamOut = new MemoryStream();
bigStream.CopyTo(bigStreamOut);
context.Body = bigStreamOut.ToArray();
}
return Task.FromResult(0);
}
/// <summary>
/// A FileInfo extension method that extracts the g zip.
/// </summary>
/// <param name="this">The @this to act on.</param>
/// <example>
/// <code>
/// using System;
/// using System.IO;
/// using Microsoft.VisualStudio.TestTools.UnitTesting;
/// using Z.ExtensionMethods;
///
/// namespace ExtensionMethods.Examples
/// {
/// [TestClass]
/// public class System_IO_FileInfo_ExtractGZip
/// {
/// [TestMethod]
/// public void ExtractGZip()
/// {
/// // Type
/// var @this = new FileInfo(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Examples_System_IO_FileInfo_ExtractGZip.txt"));
/// var output = new FileInfo(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Examples_System_IO_FileInfo_ExtractGZip.gz"));
/// var outputExtract = new FileInfo(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Examples_System_IO_FileInfo_ExtractGZip_Example.txt"));
///
/// // Intialization
/// using (FileStream stream = @this.Create())
/// {
/// }
///
/// // Examples
/// @this.CreateGZip(output);
/// output.ExtractGZip(outputExtract);
///
/// // Unit Test
/// Assert.IsTrue(outputExtract.Exists);
/// }
/// }
/// }
/// </code>
/// </example>
public static void ExtractGZip(this FileInfo @this)
{
using (FileStream originalFileStream = @this.OpenRead())
{
string newFileName = Path.GetFileNameWithoutExtension(@this.FullName);
using (FileStream decompressedFileStream = File.Create(newFileName))
{
using (var decompressionStream = new GZipStream(originalFileStream, CompressionMode.Decompress))
{
decompressionStream.CopyTo(decompressedFileStream);
}
}
}
}
public static bool DecompressFile(string pathToCompressedFile, string pathToDecompressedFile)
{
if (File.Exists(pathToCompressedFile))
{
using (FileStream inFile = File.Open(pathToCompressedFile, FileMode.Open))
{
using (FileStream outFile = File.Create(pathToDecompressedFile))
{
using (GZipStream Decompress = new GZipStream(inFile,
CompressionMode.Decompress))
{
Decompress.CopyTo(outFile);
}
}
}
}
else
{
return false;
}
return true;
}
static void Mainy(string[] args)
{
//return;
int upper = int.MaxValue;
int lower = 1;
var sw = new Stopwatch();
var boe23 = SuperfastSpecialPrimes.Primer.Solve(1, int.MaxValue).ToArray();
var formatter = new BinaryFormatter();
var fs = new FileStream(@"c:\temp\SpecialPrimeInts.dat", FileMode.Create);
formatter.Serialize(fs, boe23);
for (var n = 0; n < 10; n++)
{
sw.Start();
SuperfastSpecialPrimes.Primer.Solve(1, int.MaxValue - 4);
sw.Stop();
Console.WriteLine($"took {sw.ElapsedMilliseconds} ms");
sw.Reset();
sw.Start();
SuperfastSpecialPrimes.Primer.Solve(1, int.MaxValue - 4);
sw.Stop();
Console.WriteLine($"2 took {sw.ElapsedMilliseconds} ms");
sw.Reset();
}
return;
//var sp = new SpecialPrimeFinder(PrimeFinderFactory.CreatePrimeFinder(lower, upper));
//int boe = 0;
//var oba = new OddBitArray(lower,upper);
//foreach (int i in sp.GetSpecialPrimes(lower, upper))
//{
// oba[i] = true;
// boe++;
//};
//sw.Stop();
//Console.WriteLine($"found {boe} took {sw.ElapsedMilliseconds} ms");
//sw.Reset();
// var formatter = new BinaryFormatter();
// var fs = new FileStream(@"c:\temp\SpecialPrimeOddBitArray.dat", FileMode.Create);
// formatter.Serialize(fs, oba);
// Assembly.GetExecutingAssembly()
// GetManifestoResourceStream()
//OddBitArray boe;
////var formatter = new BinaryFormatter();
//using (Stream stream = new FileStream(@"c:\temp\SpecialPrimeOddBitArray.dat", FileMode.Open))
//{
// boe = formatter.Deserialize(stream) as OddBitArray;
//};
//var oba1 = new SuperfastSpecialPrimes.OddBitArray(1, int.MaxValue);
//for (int i = 1; i <= int.MaxValue && i > 0; i = i + 2)
//{
// oba1[i] = boe[i];
//}
//using (var fs = new FileStream(@"c:\temp\SpecialPrimeOddBitArrayBig.dat", FileMode.Create))
//{
// formatter.Serialize(fs, oba1);
//}
//return;
//using (var ms = new MemoryStream())
//{
// formatter.Serialize(ms, boe);
// ms.Position = 0;
// using (FileStream compressedFileStream = File.Create(@"c:\temp\SpecialPrimeOddBitArray3.dat"))
// {
// using (GZipStream compressionStream = new GZipStream(compressedFileStream,
// CompressionMode.Compress))
// {
// ms.CopyTo(compressionStream);
// compressionStream.Close();
// }
// }
//}
var outputStream = new MemoryStream();
using (var inputStream = File.Open(@"c:\temp\SpecialPrimeOddBitArray3.dat", FileMode.Open, FileAccess.Read, FileShare.Read))
using (var gzip = new GZipStream(inputStream, CompressionMode.Decompress))
{
gzip.CopyTo(outputStream);
gzip.Close();
inputStream.Close();
// After writing to the MemoryStream, the position will be the size
// of the decompressed file, we should reset it back to zero before returning.
outputStream.Position = 0;
}
var boe2 = formatter.Deserialize(outputStream) as OddBitArray;
}
private void HandleMessage(JObject obj)
{
Debug.Assert(obj != null, nameof(obj) + " != null");
var uri = obj["uri"]?.ToString();
var headers = GetHeaders(obj);
var payloads = (JArray)obj["payloads"];
byte[] decodedPayload;
if (payloads != null)
{
if (headers.ContainsKey("Content-Type") &&
(headers["Content-Type"].Equals("application/json") ||
headers["Content-Type"].Equals("text/plain")))
{
if (payloads.Count > 1)
{
throw new InvalidOperationException();
}
decodedPayload = Encoding.Default.GetBytes(payloads[0].ToString());
}
else
{
var payloadsStr = new string[payloads.Count];
for (var i = 0; i < payloads.Count; i++)
{
payloadsStr[i] = payloads[i].ToString();
}
var x = string.Join("", payloadsStr);
using var @in = new MemoryStream();
using var outputStream = new MemoryStream(Convert.FromBase64String(x));
if (headers.ContainsKey("Transfer-Encoding") &&
(headers["Transfer-Encoding"]?.Equals("gzip") ?? false))
{
using var decompressionStream = new GZipStream(outputStream, CompressionMode.Decompress);
decompressionStream.CopyTo(@in);
Debug.WriteLine($"Decompressed");
}
decodedPayload = @in.ToArray();
}
}
else
{
decodedPayload = new byte[0];
}
var interesting = false;
lock (_msgListeners)
{
foreach (var listener in _msgListeners.Keys)
{
var dispatched = false;
var keys = _msgListeners[listener];
foreach (var key
in
keys.Where(key => uri != null && uri.StartsWith(key) && !dispatched))
{
interesting = true;
listener.OnMessage(uri !, headers, decodedPayload);
dispatched = true;
}
}
}
if (!interesting)
{
Debug.WriteLine("Couldn't dispatch message: " + uri);
}
}
public override async Task <string> ProcessMessageAsync(KinesisEvent kinesis)
{
// NOTE (2018-12-11, bjorg): this function is responsible for error logs parsing; therefore, it CANNOT error out itself;
// instead, it must rely on aggressive exception handling and redirect those message where appropriate.
try {
foreach (var record in kinesis.Records)
{
try {
LogEventsMessage message;
using (var stream = new MemoryStream()) {
using (var gzip = new GZipStream(record.Kinesis.Data, CompressionMode.Decompress)) {
gzip.CopyTo(stream);
stream.Position = 0;
}
message = DeserializeJson <LogEventsMessage>(stream);
}
// skip events from own module
if (message.LogGroup.Contains(Info.FunctionName))
{
LogInfo("skipping event from own event log");
continue;
}
// skip control messages
if (message.MessageType == "CONTROL_MESSAGE")
{
LogInfo("skipping control message");
continue;
}
if (message.MessageType != "DATA_MESSAGE")
{
LogWarn("unknown message type: {0}", message.MessageType);
continue;
}
if (!message.LogGroup.StartsWith(LOG_GROUP_PREFIX, StringComparison.Ordinal))
{
LogWarn("unexpected log group: {0}", message.LogGroup);
continue;
}
// process messages
var functionId = message.LogGroup.Substring(LOG_GROUP_PREFIX.Length);
LogInfo($"getting owner for function: {functionId}");
var owner = await GetOwnerMetaDataAsync($"F:{functionId}");
if (owner != null)
{
foreach (var entry in message.LogEvents)
{
if (!await _logic.ProgressLogEntryAsync(owner, entry.Message, entry.Timestamp))
{
LogWarn("Unable to parse message: {0}", entry.Message);
}
}
}
else
{
LogWarn("unable to retrieve registration for: {0}", message.LogGroup);
}
} catch (Exception e) {
await SendErrorExceptionAsync(e);
}
}
return("Ok");
} catch (Exception e) {
await SendErrorExceptionAsync(e);
return($"Error: {e.Message}");
}
}
public void Reallocate_PSPPCompressedFile_CopiesAllReordsAndVariables()
{
//after this test the new file is the same size as the original
int originalRecordCount;
int originalVariableCount;
using (FileStream fileStream = new FileStream(@"TestFiles/34mbfile.sav.gz", FileMode.Open,
FileAccess.Read, FileShare.Read, 2048 * 10, FileOptions.SequentialScan))
using (GZipStream deCompressStream = new GZipStream(fileStream, CompressionMode.Decompress, true))
using (MemoryStream memStream = new MemoryStream())
{
deCompressStream.CopyTo(memStream);
memStream.Position = 0;
SpssReader spssDataset = new SpssReader(memStream);
originalRecordCount = spssDataset.Records.Count();
originalVariableCount = spssDataset.Variables.Count();
var deallocator = new DataReallocator();
// code under test
var records = deallocator.ReAllocateRecords(spssDataset.Records);
var variables = spssDataset.Variables
.Where(x => x.Name == "qhidimpconcepteval" || x.Name == "qhidimpproduct1")
.ToList();
var datalist = new List <string>();
foreach (var record in records)
{
foreach (var variable in variables)
{
var data = record.GetValue(variable)?.ToString()?.Trim();
if (!string.IsNullOrEmpty(data))
{
datalist.Add(data);
}
}
}
using (FileStream outputFileStream = new FileStream(@"4mbfile-after-reallocation.sav", FileMode.Create, FileAccess.Write))
{
using (var writer = new SpssWriter(outputFileStream, spssDataset.Variables, new SpssOptions()))
{
foreach (var record in records)
{
var newRecord = writer.CreateRecord(record);
writer.WriteRecord(newRecord);
}
writer.EndFile();
}
}
}
var results = FileValidator.CheckSpssFile(@"4mbfile-after-reallocation.sav");
Assert.AreEqual(originalRecordCount, results.RecordCount);
Assert.AreEqual(originalVariableCount, results.VariableCount);
}
/// <summary>
/// Returns the uncompressed content from a response that
/// is supposed to return text based on its content type.
///
/// If the incoming data is already uncompressed then the
/// content is returned as is.
///
/// Returns the uncompressed content that arrived in the
/// form of data compressed by gzip or deflate.
///
/// Returns an empty string if the content type is not
/// recognized as text.
///
/// Returns an error message if an exception is thrown.
/// </summary>
/// <param name="response">The response to a request</param>
public static string GetUncompressedTextContent(HttpWebResponse response)
{
string content = "";
if (ResponseCategory(response) != FileTools.TEXT)
{
return(content);
}
string encoding = response.ContentEncoding ?? "";
try
{
if (encoding == "")
{
using (Stream stream = response.GetResponseStream())
{
using (StreamReader reader = new StreamReader(stream))
{
content = reader.ReadToEnd();
}
}
}
}
catch (Exception ex)
{
content = "Error in obtaining uncompressed data "
+ ex.Message;
}
try
{
if (encoding == "gzip")
{
var response_gzip = response.GetResponseStream();
using (MemoryStream stream = new MemoryStream())
{
using (GZipStream uncompress =
new GZipStream
(response_gzip, CompressionMode.Decompress))
{
uncompress.CopyTo(stream);
content = Encoding.UTF8.GetString(stream.ToArray());
}
}
}
}
catch (Exception ex)
{
content = "Error in obtaining gzip compressed data "
+ ex.Message;
}
try
{
if (encoding == "deflate")
{
var response_deflate = response.GetResponseStream();
using (MemoryStream stream = new MemoryStream())
{
using (DeflateStream uncompress =
new DeflateStream
(response_deflate, CompressionMode.Decompress))
{
uncompress.CopyTo(stream);
content = Encoding.UTF8.GetString(stream.ToArray());
}
}
}
}
catch (Exception ex)
{
content = "Error in obtaining deflate compressed data "
+ ex.Message;
}
return(content);
}
private static void RunTests(List <Test> tests, string token, string userName, string password)
{
foreach (var test in tests)
{
var requestUri = test.RequestUri;
var qs = new List <Tuple <string, string> >();
if (test.HttpMethod == "get")
{
qs.Add(new Tuple <string, string>("query", test.Query));
}
qs.Add(new Tuple <string, string>("gzip", test.GZip.ToString()));
for (int i = 0; i < qs.Count; i++)
{
var q = qs[i];
string prefix = i > 0 ? "&" : "?";
requestUri += $"{prefix}{q.Item1}={q.Item2}";
}
var req = WebRequest.CreateHttp(requestUri);
req.Method = test.HttpMethod;
Console.WriteLine($"Running Test {test.HttpMethod}: {requestUri} with auth {test.AuthScheme} via {test.HttpMethod} ");
if (test.AuthScheme == "bearer")
{
req.Headers.Add("Authorization", $"bearer {token}");
}
else if (test.AuthScheme == "basic")
{
//System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(auth));
var cred = $"{userName}:{password}";
var auth = Convert.ToBase64String(Encoding.UTF8.GetBytes(cred));
req.Headers.Add("Authorization", $"basic {auth}");
}
Console.WriteLine($" HTTP Request Headers");
foreach (var h in req.Headers.AllKeys)
{
Console.WriteLine($" {h}: {req.Headers[h].Substring(0,50)}");
}
if (test.HttpMethod == "post")
{
using (var sr = new StreamWriter(req.GetRequestStream()))
{
sr.Write(test.Query);
}
}
HttpWebResponse resp;
try
{
resp = (HttpWebResponse)req.GetResponse();
}
catch (WebException ex)
{
Console.WriteLine(ex.Message);
resp = (HttpWebResponse)ex.Response;
if (resp == null)
{
throw ex;
}
}
Console.WriteLine($"Response {(int)resp.StatusCode}({resp.StatusDescription})");
for (int i = 0; i < resp.Headers.Count; ++i)
{
Console.WriteLine("{0}: {1}", resp.Headers.Keys[i], resp.Headers[i]);
}
Console.WriteLine();
var ms = new MemoryStream();
if (resp.ContentEncoding == "gzip")
{
using (var gs = new GZipStream(resp.GetResponseStream(), CompressionMode.Decompress))
{
gs.CopyTo(ms);
}
}
else
{
resp.GetResponseStream().CopyTo(ms);
}
ms.Position = 0;
var bytes = ms.ToArray();
Console.WriteLine($"Body Length {ms.Length}: Starts [{Encoding.UTF8.GetString(bytes,0,Math.Min(1000,bytes.Length))} . . .]");
if (resp.StatusCode == test.ExpectedResponse)
{
var color = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Test passed.");
Console.ForegroundColor = color;
}
else
{
var color = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine($"Test Failed. Expected HTTP Response code:{test.ExpectedResponse} got:{resp.StatusCode}");
Console.ForegroundColor = color;
}
//if (resp.ContentEncoding != "gzip")
//{
// var body = Encoding.UTF8.GetString(ms.ToArray());
// Console.WriteLine(body);
//}
Console.WriteLine();
}
}
public async Task <byte[]> GetObject(ObjectKey key)
{
Stopwatch timer = Stopwatch.StartNew();
Func <HttpRequestMessage> request = () => new HttpRequestMessage
{
RequestUri = GetObjectUri(key),
Method = HttpMethod.Get,
Headers = { { "Date", DateTimeOffset.UtcNow.ToString("r") } }
};
string contentType = null;
long? contentLength = null;
long? objectLength = null;
try
{
bool isCompressed;
MemoryStream responseBuffer;
using (HttpResponseMessage response = await SendAsync(request))
{
if (!response.IsSuccessStatusCode)
{
S3Error error = await DecodeError(response);
LogOperation("Get", key, null, null, null, timer, OperationStatus.Error, error.ResponseBody);
if (error.Code == "NoSuchKey")
{
return(null);
}
if (error.Code == "AccessDenied")
{
return(null);
}
throw new S3Exception("Get", key, error);
}
contentType = response.Content.Headers.ContentType.ToString();
contentLength = response.Content.Headers.ContentLength ?? 0;
objectLength = contentLength;
string ol = response.Headers.Value(OriginalLengthHeader);
if (!String.IsNullOrWhiteSpace(ol))
{
objectLength = long.Parse(ol);
}
responseBuffer = new MemoryStream();
await response.Content.CopyToAsync(responseBuffer);
responseBuffer.Position = 0;
isCompressed = response.Content.Headers.ContentEncoding.Contains("gzip");
}
byte[] data = new byte[objectLength.Value];
if (isCompressed)
{
var sourceStream = new GZipStream(responseBuffer, CompressionMode.Decompress);
var targetStream = new MemoryStream(data);
sourceStream.CopyTo(targetStream);
}
else
{
responseBuffer.Read(data, 0, data.Length);
}
LogOperation(
"Get",
key,
contentType,
objectLength,
contentLength,
timer,
OperationStatus.OK,
null);
return(data);
}
catch (S3Exception) { throw; }
catch (Exception e)
{
LogOperation(
"Get",
key,
contentType,
objectLength,
contentLength,
timer,
OperationStatus.Exception,
e.ToString());
throw;
}
}
public NRRD(TextAsset asset)
{
using (var reader = new BinaryReader(new MemoryStream(asset.bytes))) {
for (string line = reader.ReadLine(); line.Length > 0; line = reader.ReadLine())
{
if (line.StartsWith("#") || !line.Contains(":"))
{
continue;
}
var tokens = line.Split(':');
var key = tokens[0].Trim();
var value = tokens[1].Trim();
headers.Add(key, value);
}
if (headers["dimension"] != "3")
{
throw new ArgumentException("NRRD is not 3D");
}
if (headers["type"] != "float")
{
throw new ArgumentException("NRRD is not of type float");
}
if (headers["endian"] != "little")
{
throw new ArgumentException("NRRD is not little endian");
}
if (headers["encoding"] != "gzip")
{
throw new ArgumentException("NRRD is not gzip encoded");
}
dims = Array.ConvertAll(headers["sizes"].Split(), s => int.Parse(s));
if (headers.ContainsKey("space origin"))
{
var origin = Array.ConvertAll(headers["space origin"].Substring(1, headers["space origin"].Length - 2).Split(','), v => float.Parse(v));
this.origin = new Vector3(origin[0], origin[1], origin[2]);
}
if (headers.ContainsKey("space directions"))
{
var scale = Array.ConvertAll(headers["space directions"].Split(), s => Array.ConvertAll(s.Substring(1, s.Length - 2).Split(','), v => float.Parse(v)));
if (scale[0][0] == 0 || scale[1][1] == 0 || scale[2][2] == 0)
{
throw new ArgumentException("NRRD has 0 scale value");
}
if (scale[0][1] != 0 || scale[1][0] != 0 || scale[2][0] != 0 ||
scale[0][2] != 0 || scale[1][2] != 0 || scale[2][1] != 0)
{
throw new ArgumentException("NRRD is not axis-aligned");
}
this.scale = new Vector3(scale[0][0], scale[1][1], scale[2][2]);
}
var mem = new MemoryStream();
using (var stream = new GZipStream(reader.BaseStream, CompressionMode.Decompress)) stream.CopyTo(mem);
data = new float[dims[0] * dims[1] * dims[2]];
Buffer.BlockCopy(mem.ToArray(), 0, data, 0, data.Length * sizeof(float));
}
}
public void DownloadToDisk___(Socket client, string FileURL, string LocalFilePath)
{
Uri url = new Uri(FileURL);
IPAddress[] addresses = System.Net.Dns.GetHostAddresses(url.Host);
IPEndPoint hostep = new IPEndPoint(addresses[0], 80);
Socket sock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
sock.Connect(hostep);
string request_txt = // File.ReadAllText("req.txt");
string.Format("GET {0} HTTP/1.1\r\n", url.ToString()) +
string.Format("Host: {0}\r\n", url.Host) +
"Connection: keep-alive\r\n" +
"Upgrade-Insecure-Requests: 1\r\n" +
"User-Agent: Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36\r\n" +
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\n" +
"Accept-Encoding: gzip, deflate, sdch\r\n" +
"Accept-Language: en-US,en;q=0.8\r\n" +
"Cache-Control: no-cache\r\n\r\n";
// request_txt = string.Format(request_txt, url.ToString(), url.Host);
int response = sock.Send(Encoding.UTF8.GetBytes(request_txt));
const int MAX_BYTES_RECEIVED = 10000000; // 10M max image size.
byte[] bytesReceived = new byte[MAX_BYTES_RECEIVED];
int n_bytes_received = sock.Receive(bytesReceived, bytesReceived.Length, SocketFlags.None);
string packet_str = System.Text.Encoding.Default.GetString(bytesReceived, 0, n_bytes_received);
int start_of_data_i = packet_str.IndexOf("\r\n\r\n") + 4;
int content_length = int.Parse(Regex.Match(packet_str, @"^Content-Length:\s+(?<len>\d+)\s*$", RegexOptions.Multiline).Groups["len"].Value);
Console.WriteLine("content_length {0}", content_length);
bool is_gzip = Regex.IsMatch(packet_str, @"Content-Encoding:.*\bgzip\b", RegexOptions.Multiline);
int retries = 0;
while (content_length == 0 || n_bytes_received - start_of_data_i < content_length)
{
sock.ReceiveTimeout = 10000;
int n = 0;
try
{
n = sock.Receive(bytesReceived, n_bytes_received, bytesReceived.Length - n_bytes_received, SocketFlags.None);
}
catch (Exception e)
{
if (retries++ < 2)
{
continue;
}
break;
}
if (n == 0)
{
break;
}
n_bytes_received += n;
}
sock.Close();
byte[] res_data = new byte[n_bytes_received - start_of_data_i];
Array.Copy(bytesReceived, start_of_data_i, res_data, 0, n_bytes_received - start_of_data_i);
if (is_gzip)
{
GZipStream decompressionStream = new GZipStream(new MemoryStream(res_data), CompressionMode.Decompress);
MemoryStream ms = new MemoryStream();
decompressionStream.CopyTo(ms);
res_data = ms.ToArray();
}
Stream dataStream = new MemoryStream(res_data);
using (var fs = new FileStream(LocalFilePath, FileMode.Create, FileAccess.Write))
dataStream.CopyTo(fs);
}
public void _request(string endPoint, RequestData requestData)
{
requestData.timestamp = PredictedServerTimestamp;
requestData.version = Version;
requestData.versionKey = VersionKey;
LastRequest = new LastRequest();
LastRequest.EndPoint = endPoint;
LastRequest.RequestData = requestData;
Byte[] payload = Encoding.UTF8.GetBytes(JsonUtility.ToJson(requestData));
if (localSendInvalidRequestData)
{
Array.Clear(payload, 0, payload.Length);
}
payload = _encryptionHelper.Encrypt(payload, this.nonce);
string payloadBase64 = Convert.ToBase64String(payload);
var postForm = new RawJsonForm();
postForm.AddField("payloadBase64", payloadBase64);
// signature
Byte[] digest = md5.ComputeHash(payload); // md5(payload);
Byte[] nonceByte = BitConverter.GetBytes(this.nonce); // nonce to byte
Byte[] signMessage = new Byte[digest.Length + nonceByte.Length]; // nonceByte + digest
Array.Copy(nonceByte, signMessage, nonceByte.Length);
Array.Copy(digest, 0, signMessage, nonceByte.Length, digest.Length);
signMessage = _encryptionHelper.SignMessage(signMessage);
string signedBase64 = Convert.ToBase64String(signMessage);
if (localSendInvalidSignBase64)
{
signedBase64 = "1234";
}
// query
Dictionary <string, string> queryDict = new Dictionary <string, string>();
queryDict.Add("signedBase64", signedBase64);
queryDict.Add("token", token);
if (remoteTimeout)
{
queryDict.Add("remoteTimeout", "true");
}
if (remoteSendInvalidPayload)
{
queryDict.Add("remoteSendInvalidPayload", "true");
}
var query = this.getQuery(queryDict);
// send request
var url = $"{Host}{endPoint}?{query}";
var nonceSent = nonce;
HTTPRequest request = new HTTPRequest(new Uri(url), HTTPMethods.Post, (originalRequest, response) =>
{
isSending = false;
try
{
if (response != null && response.IsSuccess)
{
var isEncrypted = false;
if (response.Headers.TryGetValue("content-encoding", out var isEncryptString))
{
if (isEncryptString.Count > 0 && isEncryptString[0] == "encrypted")
{
isEncrypted = true;
}
}
if (isEncrypted)
{
// decrypt
var decrypted = _encryptionHelper.Decrypt(response.Data, nonceSent);
// decompress
var memoryStream = new MemoryStream(decrypted);
var toMemoryStream = new MemoryStream();
var gZipStream = new GZipStream(memoryStream, CompressionMode.Decompress);
gZipStream.CopyTo(toMemoryStream);
memoryStream.Close();
var decompressed = toMemoryStream.ToArray();
var text = Encoding.UTF8.GetString(decompressed);
// Debug.Log( $"response Data: {text}." );
RespondData = JsonUtility.FromJson <RespondData>(text);
}
else
{
RespondData = JsonUtility.FromJson <RespondData>(response.DataAsText);
}
if (!string.IsNullOrEmpty(RespondData.token))
{
token = RespondData.token;
}
if (RespondData.user.userId > 0)
{
user = RespondData.user;
}
card = RespondData.body.card;
cards = RespondData.body.cards;
serverTimestamp = RespondData.timestamp;
receivedTimestamp = CurrentTimestamp;
}
else
{
RespondData.errorCode = ErrorCode.Timeout;
}
}
catch (Exception e)
{
RespondData.errorCode = ErrorCode.FailedToDecryptServerPayload;
}
});
// request.ConnectTimeout = TimeSpan.FromSeconds( Timeout );
request.Timeout = TimeSpan.FromSeconds(Timeout);
request.SetForm(postForm);
request.Send();
isSending = true;
nonce++;
}
private static void DecryptFile(string password, string inputFile, string outputFile)
{
try
{
var sw = new Stopwatch();
using (FileStream fsIn = new FileStream(inputFile, FileMode.Open, FileAccess.Read, FileShare.Read, BUFFER_SIZE))
{
var sig = new byte[3];
var expectedSig = Encoding.UTF8.GetBytes("GCM");
var versionMajor = new byte[1];
var versionMinor = new byte[1];
var salt = new byte[SALT_LENGTH];
var key2Encrypted = new byte[KEY_LENGTH];
var key2EncryptedTag = new byte[TAG_LENGTH];
var compressed = new byte[1];
var BEchunkSize = new byte[4];
int headerLength;
var headerTag = new byte[TAG_LENGTH];
fsIn.ForceRead(sig, 0, sig.Length);
fsIn.ForceRead(versionMajor, 0, versionMajor.Length);
fsIn.ForceRead(versionMinor, 0, versionMinor.Length);
if (!sig.SequenceEqual(expectedSig) ||
versionMajor[0] != 1 ||
versionMinor[0] != 1)
{
Console.WriteLine("Unsupported input file version");
return;
}
else
{
headerLength = V1_1_HEADER_LENGTH;
}
//Read in rest of V1.0 header pieces
fsIn.ForceRead(salt, 0, salt.Length);
fsIn.ForceRead(key2Encrypted, 0, key2Encrypted.Length);
fsIn.ForceRead(key2EncryptedTag, 0, key2EncryptedTag.Length);
fsIn.ForceRead(compressed, 0, compressed.Length);
fsIn.ForceRead(BEchunkSize, 0, BEchunkSize.Length);
//But then read full header in one chunk, and then tag, to authenticate it before continuing
var header = new byte[headerLength];
fsIn.Position = 0;
fsIn.ForceRead(header, 0, headerLength);
fsIn.ForceRead(headerTag, 0, headerTag.Length);
Rfc2898DeriveBytes k1 = new Rfc2898DeriveBytes(password, salt, 10000, HashAlgorithmName.SHA256);
byte[] key1 = k1.GetBytes(32);
GcmDecrypt(NO_DATA, key1, HEADER_NONCE, headerTag, header);
byte[] key2 = GcmDecrypt(key2Encrypted, key1, FEK_NONCE, key2EncryptedTag);
int chunkSize = BigEndianBytesToInt(BEchunkSize);
bool compression = compressed[0] == 1 ? true : false;
using (FileStream fsOut = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.None, BUFFER_SIZE))
using (var ms = new MemoryStream())
using (GZipStream gstr = compression ? new GZipStream(ms, CompressionMode.Decompress) : null)
{
sw.Start();
if (compression)
{
ChunkedDecrypt(key2, chunkSize, fsIn, ms);
ms.Position = 0;
gstr.CopyTo(fsOut);
}
else
{
ChunkedDecrypt(key2, chunkSize, fsIn, fsOut);
}
sw.Stop();
Console.WriteLine("File decrypted. AES GCM decryption took {0} ms", sw.ElapsedMilliseconds);
}
}
}
catch (Exception ex)
{
Console.WriteLine($"Decryption failed: {ex.Message}");
}
}
public static void Run(string[] args)
{
if (!Directory.Exists(outputDirectory))
{
Directory.CreateDirectory(outputDirectory);
}
Dictionary <string, string> generatedBlocks = new();
ushort maxStateId = 0;
MemoryStream compressedStream = RMSManager.Get().GetStream(Properties.Resources.blocks);
GZipStream zipStream = new(compressedStream, CompressionMode.Decompress);
MemoryStream resultStream = RMSManager.Get().GetStream();
zipStream.CopyTo(resultStream);
dynamic blocks = JSON.DeserializeDynamic(Encoding.UTF8.GetString(resultStream.ToArray()));
compressedStream = RMSManager.Get().GetStream(Properties.Resources.blockstates);
zipStream = new GZipStream(compressedStream, CompressionMode.Decompress);
resultStream = RMSManager.Get().GetStream();
zipStream.CopyTo(resultStream);
dynamic blockstates = JSON.DeserializeDynamic(Encoding.UTF8.GetString(resultStream.ToArray()));
int bitsPerBlock = 0;
foreach (dynamic block in blocks)
{
string namedId = block.Key;
// air is unneeded
if (namedId == "minecraft:air")
{
continue;
}
Console.WriteLine("Generating " + namedId);
dynamic block1 = blockstates[0].blocks.block[namedId.Replace("minecraft:", "")];
if (block1 == null)
{
Console.WriteLine("eh?? " + namedId);
continue;
}
bool propertiesDone = false;
int protocolId = block1.numeric_id;
int minState = block1.min_state_id;
int maxState = block1.max_state_id;
int defaultState = 0;
if (maxState > maxStateId)
{
maxStateId = (ushort)maxState;
}
Dictionary <string, string> properties = new();
StringBuilder stateGet = new();
StringBuilder stateSet = new();
StringBuilder parameters = new();
StringBuilder setters = new();
bool hasProperties = false;
foreach (dynamic state in block.Value.states)
{
StringBuilder checks = new();
if (state.properties != null)
{
hasProperties = true;
foreach (dynamic property in state.properties)
{
checks.Append(((string)property.Key).ToPascalCase());
checks.Append(" == ");
if (property.Value == "true" || property.Value == "false")
{
checks.Append((string)property.Value);
if (!propertiesDone)
{
properties.Add((string)property.Key, string.Format(propertyTemplate, "bool",
((string)property.Key).ToPascalCase()));
parameters.Append(string.Format(parameterTemplate, "bool", property.Key));
setters.Append(string.Format(setterTemplate, ((string)property.Key).ToPascalCase(),
property.Key));
}
}
else if (((string)property.Value).All(char.IsDigit))
{
checks.Append((string)property.Value);
if (!propertiesDone)
{
properties.Add((string)property.Key, string.Format(propertyTemplate, "int",
((string)property.Key).ToPascalCase()));
parameters.Append(string.Format(parameterTemplate, "int", property.Key));
setters.Append(string.Format(setterTemplate, ((string)property.Key).ToPascalCase(),
property.Key));
}
}
else if (decimal.TryParse((string)property.Value, out _))
{
checks.Append((string)property.Value);
if (!propertiesDone)
{
properties.Add((string)property.Key, string.Format(propertyTemplate, "float",
((string)property.Key).ToPascalCase()));
parameters.Append(string.Format(parameterTemplate, "float", property.Key));
setters.Append(string.Format(setterTemplate, ((string)property.Key).ToPascalCase(),
property.Key));
}
}
else
{
//checks.Append('"');
checks.Append(property.Value);
//checks.Append('"');
if (!propertiesDone)
{
properties.Add((string)property.Key, string.Format(propertyTemplate, "string",
((string)property.Key).ToPascalCase()));
parameters.Append(string.Format(parameterTemplate, "string", property.Key));
setters.Append(string.Format(setterTemplate, ((string)property.Key).ToPascalCase(),
property.Key));
}
}
checks.Append(" && ");
}
}
if (state.@default != null)
{
defaultState = state.id;
foreach (string key in properties.Keys.ToList())
{
if (state.properties[key] == "true" || state.properties[key] == "false")
{
properties[key] += (string)state.properties[key] + ";";
}
else if (((string)state.properties[key]).All(char.IsDigit))
{
properties[key] += (string)state.properties[key] + ";";
}
else if (decimal.TryParse((string)state.properties[key], out _))
{
properties[key] += (string)state.properties[key] + ";";
}
else
{
properties[key] += "\"" + (string)state.properties[key] + "\";";
}
}
}
if (!propertiesDone)
{
propertiesDone = true;
}
if (hasProperties)
{
string checks1 = checks.ToString();
checks1 = checks1.Substring(0, checks1.Length - 4);
stateGet.Append(string.Format(ifTemplate, checks1, "return " + state.id + ";"));
checks1 = checks1.Replace("==", "=");
checks1 = checks1.Replace(" && ", ";\n");
checks1 += ";";
stateSet.Append(string.Format(ifTemplate, "value == " + state.id, checks1));
}
}
StringBuilder propertiesStr = new();
foreach (KeyValuePair <string, string> entry in properties)
{
propertiesStr.Append(entry.Value);
propertiesStr.Append('\n');
}
if (hasProperties)
{
parameters.Length -= 2;
}
string b = string.Format(hasProperties ? blockTemplate : blockTemplate0, namedId.Replace("minecraft:", "").ToPascalCase(),
namedId, minState, maxState, defaultState, stateGet, stateSet, propertiesStr, parameters, setters, protocolId);
generatedBlocks.Add("Block" + namedId.Replace("minecraft:", "").ToPascalCase() + ".cs", b);
}
Console.WriteLine("Saving blocks...");
foreach (KeyValuePair <string, string> generated in generatedBlocks)
{
File.WriteAllText(Path.Combine(outputDirectory, generated.Key),
generated.Value);
}
bitsPerBlock = (int)Math.Ceiling(Math.Log2(maxStateId));
Console.WriteLine("Bits per block: " + bitsPerBlock);
}
/// <summary>
/// Unpacks files from an APAK archive
/// </summary>
/// <param name="Input">Input stream. Must be set to the start of the header. Left open after completion</param>
/// <param name="OutputDirectory">Output directory. If <see cref="null"/>, the file is parsed but no extraction is performed</param>
/// <param name="CollectFiles"><see cref="true"/> to collect and return FileSpecs</param>
/// <param name="Log">Logger for messages</param>
/// <returns>File Specs if <paramref name="CollectFiles"/> is <see cref="true"/>, otherwise <see cref="null"/></returns>
/// <remarks>
/// Using <paramref name="OutputDirectory"/>=<see cref="null"/> and <paramref name="CollectFiles"/>=<see cref="true"/> will return files only, not extract.
/// It's recommended to Leave <paramref name="CollectFiles"/> set to <see cref="false"/> to avoid memory problems when extracting pak files with millions of entries.
/// </remarks>
public static FileSpec[] Unpack(Stream Input, string OutputDirectory = null, bool CollectFiles = false, TextWriter Log = null)
{
if (Log == null)
{
Log = TextWriter.Null;
}
if (!Input.CanRead)
{
throw new ArgumentException("Input must be readable");
}
//This combination would do nothing and report nothing
if (string.IsNullOrWhiteSpace(OutputDirectory) && !CollectFiles)
{
throw new ArgumentException("Either 'OutputDirectory' or 'CollectFiles' must be set (or both)");
}
string RootDirectory = string.IsNullOrWhiteSpace(OutputDirectory) ? "." : Path.GetFullPath(OutputDirectory);
CompressionType Compression;
FileSpec[] Files;
int FileCount;
using (var BR = new BinaryReader(Input, Encoding.UTF8, true))
{
var Header = Encoding.ASCII.GetString(BR.ReadBytes(Encoding.ASCII.GetByteCount(HEADER)));
if (Header != HEADER)
{
throw new ArgumentException("Input stream is not an APAK file");
}
Compression = (CompressionType)BR.ReadByte();
FileCount = BR.ReadInt32();
if (FileCount < 0)
{
throw new InvalidDataException("Negative File Count speficied in Header");
}
if (CollectFiles)
{
Files = new FileSpec[FileCount];
}
else
{
Files = new FileSpec[0];
}
}
//Verify that the compression type is valid
if (!Enum.IsDefined(Compression.GetType(), Compression) || Compression == CompressionType.IndividualCompression)
{
throw new InvalidDataException($"Invalid Compression specified in File Header. Value was {Compression}");
}
//Sneakily replace source stream with GZip Stream to decompress if needed
if (Compression == CompressionType.AllCompressionGZip)
{
Input = new GZipStream(Input, CompressionMode.Decompress);
}
using (var BR = new BinaryReader(Input, Encoding.UTF8, true))
{
Log.WriteLine("Processing {0} entries", FileCount);
for (var i = 0; i < FileCount; i++)
{
var Flags = (EntryType)BR.ReadByte();
var Name = RS(BR).TrimStart('/').Replace('/', '\\');
if (string.IsNullOrWhiteSpace(Name))
{
throw new InvalidDataException("Entry Name is Empty");
}
var CurrentFile = new FileSpec(Path.GetFullPath(Path.Combine(RootDirectory, Name)), Name, Flags.HasFlag(EntryType.Directory));
if (CollectFiles)
{
Files[i] = CurrentFile;
}
Log.WriteLine("Reading {0} Dir={1}", CurrentFile.EntryName, CurrentFile.IsDirectory);
if (Flags == EntryType.Directory)
{
//Don't seek on directory entries, they have no size and data.
if (!CurrentFile.RealName.StartsWith(RootDirectory))
{
throw new InvalidDataException($"Invalid Directory Name specified. Value doen't results in s aubdirectory of {RootDirectory}. Value was {CurrentFile.RealName}");
}
Directory.CreateDirectory(CurrentFile.RealName);
}
else if (Flags.HasFlag(EntryType.File))
{
if (Flags != EntryType.File && Flags != (EntryType.File | EntryType.GZip))
{
throw new InvalidDataException($"Invalid Flag Combination for File. Value was {Flags}");
}
var Size = BR.ReadInt64();
if (Size < 0)
{
throw new InvalidDataException($"Invalid File Size specified. Value was {Size}");
}
//Extract if output has been specified
if (!string.IsNullOrWhiteSpace(OutputDirectory))
{
Log.WriteLine("Extracting File...");
var DirName = Path.GetDirectoryName(CurrentFile.RealName);
if (!DirName.StartsWith(RootDirectory))
{
throw new InvalidDataException($"Invalid File Name specified. Value doen't results in s aubdirectory of {RootDirectory}. Value was {CurrentFile.RealName}");
}
//Create directories for files too
if (!Directory.Exists(DirName))
{
Directory.CreateDirectory(DirName);
}
using (var FS = File.Create(CurrentFile.RealName))
{
if (Flags.HasFlag(EntryType.GZip))
{
//Decompress data
using (var RS = new RangedStream(Input, Size))
{
using (var GZ = new GZipStream(RS, CompressionMode.Decompress))
{
GZ.CopyTo(FS);
}
}
}
else
{
//Copy uncompressed data directly
CopyStream(Input, FS, Size);
}
}
}
else
{
//Just seek forward, we are not extracting
BR.BaseStream.Seek(Size, SeekOrigin.Current);
}
}
else
{
throw new InvalidDataException($"Unexpected Flag Combination: {Flags}");
}
}
Log.WriteLine("Done");
}
//Dispose GZip stream created in this function
if (Compression == CompressionType.AllCompressionGZip)
{
Input.Dispose();
}
return(CollectFiles ? Files : null);
}
/// <summary>
/// Executes the query as prepared statement.
/// </summary>
/// <returns>An <see cref="AceQLDataReader"/> object.</returns>
/// <exception cref="AceQL.Client.Api.AceQLException">If any Exception occurs.</exception>
private async Task <AceQLDataReader> ExecuteQueryAsPreparedStatementAsync()
{
try
{
AceQLCommandUtil aceQLCommandUtil = new AceQLCommandUtil(cmdText, Parameters);
// Get the parameters and build the result set
Dictionary <string, string> statementParameters = aceQLCommandUtil.GetPreparedStatementParameters();
// Replace all @parms with ? in sql command
cmdText = aceQLCommandUtil.ReplaceParmsWithQuestionMarks();
IFile file = await GetUniqueResultSetFileAsync().ConfigureAwait(false);
bool isStoredProcedure = (commandType == CommandType.StoredProcedure ? true : false);
bool isPreparedStatement = true;
using (Stream input = await aceQLHttpApi.ExecuteQueryAsync(cmdText, Parameters, isStoredProcedure, isPreparedStatement, statementParameters).ConfigureAwait(false))
{
if (input != null)
{
if (aceQLHttpApi.GzipResult)
{
using (GZipStream decompressionStream = new GZipStream(input, CompressionMode.Decompress))
{
using (var stream = await file.OpenAsync(PCLStorage.FileAccess.ReadAndWrite).ConfigureAwait(false))
{
decompressionStream.CopyTo(stream);
}
}
}
else
{
using (var stream = await file.OpenAsync(PCLStorage.FileAccess.ReadAndWrite).ConfigureAwait(false))
{
input.CopyTo(stream);
}
}
}
}
StreamResultAnalyzer streamResultAnalyzer = new StreamResultAnalyzer(file, aceQLHttpApi.HttpStatusCode);
if (!await streamResultAnalyzer.IsStatusOkAsync().ConfigureAwait(false))
{
throw new AceQLException(streamResultAnalyzer.GetErrorMessage(),
streamResultAnalyzer.GetErrorType(),
streamResultAnalyzer.GetStackTrace(),
aceQLHttpApi.HttpStatusCode);
}
int rowsCount = 0;
using (Stream readStreamCout = await file.OpenAsync(FileAccess.Read).ConfigureAwait(false))
{
RowCounter rowCounter = new RowCounter(readStreamCout);
rowsCount = rowCounter.Count();
}
if (isStoredProcedure)
{
using (Stream readStreamOutParms = await file.OpenAsync(FileAccess.Read).ConfigureAwait(false))
{
UpdateOutParametersValues(readStreamOutParms, Parameters);
}
}
Stream readStream = await file.OpenAsync(FileAccess.Read).ConfigureAwait(false);
AceQLDataReader aceQLDataReader = new AceQLDataReader(file, readStream, rowsCount, connection);
return(aceQLDataReader);
}
catch (Exception exception)
{
await simpleTracer.TraceAsync(exception.ToString()).ConfigureAwait(false);
if (exception.GetType() == typeof(AceQLException))
{
throw;
}
else
{
throw new AceQLException(exception.Message, 0, exception, aceQLHttpApi.HttpStatusCode);
}
}
}
protected void DownloadSub(object sender, EventArgs e)
{
var tempdir = Environment.GetEnvironmentVariable ("TEMP", EnvironmentVariableTarget.Machine);
if (string.IsNullOrEmpty (tempdir))
tempdir = "/tmp";
var selectednode = (MovieTreeNode)MovieNodeView.NodeSelection.SelectedNode;
if (selectednode == null)
return;
string movietitle = System.IO.Path.GetFileNameWithoutExtension (_fname);
var client = new WebClient ();
string filename = tempdir + "/tmp_" + movietitle + ".gz";
client.DownloadFile (selectednode.DownloadLink, filename);
var fileToDecompress = new FileInfo (filename);
string currentFileName = fileToDecompress.FullName;
string newFileName = currentFileName.Remove (currentFileName.Length - fileToDecompress.Extension.Length);
using (FileStream originalFileStream = fileToDecompress.OpenRead()) {
using (FileStream decompressedFileStream = File.Create(newFileName)) {
using (var decompressionStream = new GZipStream(originalFileStream, CompressionMode.Decompress)) {
decompressionStream.CopyTo (decompressedFileStream);
}
}
}
fileToDecompress.Delete ();
string origlocation = System.IO.Path.GetDirectoryName (_fname);
string newfile = origlocation + "/" + movietitle + "." + selectednode.SubFormat;
if (File.Exists (newfile)) {
File.Delete (newfile);
} //TODO: ask for overwrite?
File.Move (newFileName, newfile);
statusbar1.Push (3, "Downloaded " + newfile + ".");
Downloadbutton.Sensitive = false;
}
public IModel LoadModel(OutModelFileBundle modelFileBundle)
{
var outFile = modelFileBundle.OutFile;
var isBw2 = modelFileBundle.GameVersion == GameVersion.BW2;
Stream stream;
if (isBw2)
{
using var gZipStream =
new GZipStream(outFile.Impl.OpenRead(),
CompressionMode.Decompress);
stream = new MemoryStream();
gZipStream.CopyTo(stream);
stream.Position = 0;
}
else
{
stream = outFile.Impl.OpenRead();
}
using var er = new EndianBinaryReader(stream, Endianness.LittleEndian);
var bwHeightmap =
isBw2
? (IBwHeightmap)er.ReadNew <Bw2Heightmap>()
: er.ReadNew <Bw1Heightmap>();
var finModel = new ModelImpl();
var finSkin = finModel.Skin;
var finMesh = finSkin.AddMesh();
var triangles = new List <(IVertex, IVertex, IVertex)>();
var chunks = bwHeightmap.Chunks;
var heightmapWidth = 64 * 4 * 4;
var heightmapHeight = 64 * 4 * 4;
var chunkFinVertices =
new Grid <IVertex?>(heightmapWidth, heightmapHeight);
var heights = new Grid <ushort>(heightmapWidth, heightmapHeight);
for (var chunkY = 0; chunkY < chunks.Height; ++chunkY)
{
for (var chunkX = 0; chunkX < chunks.Width; ++chunkX)
{
var tiles = chunks[chunkX, chunkY]?.Tiles;
if (tiles == null)
{
continue;
}
for (var tileY = 0; tileY < tiles.Height; ++tileY)
{
for (var tileX = 0; tileX < tiles.Width; ++tileX)
{
var points = tiles[tileX, tileY].Points;
for (var pointY = 0; pointY < points.Height; ++pointY)
{
for (var pointX = 0; pointX < points.Width; ++pointX)
{
var point = points[pointX, pointY];
var heightmapX = 16 * chunkX + 4 * tileX + pointX;
var heightmapY = 16 * chunkY + 4 * tileY + pointY;
var finVertex =
finSkin.AddVertex(point.X, point.Height, point.Y)
.SetUv(1f * heightmapX / heightmapWidth,
1f * heightmapY / heightmapHeight);
chunkFinVertices[heightmapX, heightmapY] = finVertex;
heights[heightmapX, heightmapY] = point.Height;
}
}
}
}
}
}
var image = new I8Image(heightmapWidth, heightmapHeight);
image.Mutate((_, setHandler) => {
for (var vY = 0; vY < heightmapHeight; ++vY)
{
for (var vX = 0; vX < heightmapWidth; ++vX)
{
setHandler(vX, vY, (byte)(heights[vX, vY] / 24));
}
}
});
var heightmapTexture = finModel.MaterialManager.CreateTexture(image);
var finMaterial =
finModel.MaterialManager.AddTextureMaterial(heightmapTexture);
for (var vY = 0; vY < heightmapHeight - 1; ++vY)
{
for (var vX = 0; vX < heightmapWidth - 1; ++vX)
{
var a = chunkFinVertices[vX, vY];
var b = chunkFinVertices[vX + 1, vY];
var c = chunkFinVertices[vX, vY + 1];
var d = chunkFinVertices[vX + 1, vY + 1];
if (a != null && b != null && c != null && d != null)
{
triangles.Add((a, b, c));
triangles.Add((d, c, b));
}
}
}
finMesh.AddTriangles(triangles.ToArray()).SetMaterial(finMaterial);
return(finModel);
}
private void btnGenerate_Click(object sender, EventArgs e)
{
string folderPath = txtFolderPath.Text;
string contractsFilePath = folderPath + "\\contract.gz";
string securitiesFilePath = folderPath + "\\security.gz";
//if both the files does not exist then
//we will not delete the old SecurityMaster.csv file and generate new file
if (File.Exists(contractsFilePath) == false)
{
lblMessage.AppendText("contract.gz file not found at location " + folderPath + Environment.NewLine);
}
else if (File.Exists(securitiesFilePath) == false)
{
lblMessage.AppendText("security.gz file not found at location " + folderPath + Environment.NewLine);
}
else
{
// unzip the contract.gz file
// creates contract.txt file
try
{
LinkedList <string> formattedLines = new LinkedList <string>();
Dictionary <Int32, string> dictunderlyingtoken = new Dictionary <Int32, string>();
String header = "ScripNo,underlyingScripNo,Instrument,symbol,tradeSymbol,MLot,expiryDate,StrikePrice,OptType,fullname";
formattedLines.AddLast(header);
//processing the equities Security.gz file
FileInfo fiSecurity = new FileInfo(securitiesFilePath);
FileStream inFileSecurity = fiSecurity.OpenRead();
string securityZipFile = fiSecurity.FullName;
string securityNewFile = securityZipFile.Remove(securityZipFile.Length - fiSecurity.Extension.Length) + ".txt";
if (File.Exists(securityNewFile))
{
File.Delete(securityNewFile);
}
Stream outFileSecurity = File.Create(securityNewFile);
using (GZipStream Dc = new GZipStream(inFileSecurity, CompressionMode.Decompress))
{
Dc.CopyTo(outFileSecurity);
Dc.Flush();
Dc.Close();
outFileSecurity.Close();
}
lblMessage.AppendText("Unzip Security.gz Done" + Environment.NewLine);
// parse the contracts.txt and add the contents in SecurityMaster.csv
string [] lines = File.ReadAllLines(securityNewFile);
foreach (string line in lines)
{
if (line.Split('|').Length < 10)
{
continue;
}
string[] arr = line.Split('|');
if (arr[2] == "EQ")
{
string ScripNo = arr[0];
string underlyingScripNo = "";
string Instrument = arr[2];
string symbol = arr[1];
string expiryDate = "";
string StrikePrice = "";
string OptType = "";
string MLot = "";
string tradeSymbol = "";
string fullname = arr[21];
String newLine = ScripNo + "," + underlyingScripNo + "," + Instrument + "," + symbol + "," + tradeSymbol + "," + MLot + "," + expiryDate + "," + StrikePrice + "," + OptType + "," + fullname;
dictunderlyingtoken[Convert.ToInt32(ScripNo)] = fullname;
formattedLines.AddLast(newLine);
}
}
lblMessage.AppendText("Successfully processed Security.txt file" + Environment.NewLine);
FileInfo fi = new FileInfo(contractsFilePath);
FileStream inFile = fi.OpenRead();
string contractZipFile = fi.FullName;
string contractNewFile = contractZipFile.Remove(contractZipFile.Length - fi.Extension.Length) + ".txt";
if (File.Exists(contractNewFile))
{
File.Delete(contractNewFile);
}
FileStream outFile = File.Create(contractNewFile);
using (GZipStream Dc = new GZipStream(inFile, CompressionMode.Decompress))
{
Dc.CopyTo(outFile);
Dc.Flush();
Dc.Close();
outFile.Close();
}
lblMessage.AppendText("Unzip Contracts.gz Done" + Environment.NewLine);
// parse the contracts.txt and add the contents in SecurityMaster.csv
string SecurityMasterFilePath = folderPath + "\\SecurityMaster.csv";
lines = File.ReadAllLines(contractNewFile);
foreach (string line in lines)
{
if (line.Split('|').Length < 10)
{
continue;
}
string[] arr = line.Split('|');
string ScripNo = arr[0];
string underlyingScripNo = arr[1];
string Instrument = arr[2];
string symbol = arr[3];
long val = Convert.ToUInt32(arr[6]);
DateTime dt = new DateTime((val * 1000L) + TimeOffset);
double ticks = double.Parse(arr[6]) * 1000;
TimeSpan time = TimeSpan.FromMilliseconds(ticks);
DateTime startdate = new DateTime(1980, 1, 1) + time;
string expiryDate = startdate.ToString("yyyy-MM-dd");
if (arr[7] != "-1")
{
arr[7] = (Convert.ToInt32(arr[7]) / 100).ToString();
}
string StrikePrice = arr[7];
string OptType = arr[8];
string MLot = arr[30];
string tradeSymbol = arr[53];
string fullname = tradeSymbol;
if (dictunderlyingtoken.ContainsKey(Convert.ToInt32(underlyingScripNo)))
{
fullname = dictunderlyingtoken[Convert.ToInt32(underlyingScripNo)];
}
else
{
lblMessage.AppendText("Token not Found : " + underlyingScripNo);
}
String newLine = ScripNo + "," + underlyingScripNo + "," + Instrument + "," + symbol + "," + tradeSymbol + "," + MLot + "," + expiryDate + "," + StrikePrice + "," + OptType + "," + fullname;
formattedLines.AddLast(newLine);
}
lblMessage.AppendText("Successfully processed contract.txt file" + Environment.NewLine);
string[] the_array = formattedLines.Select(i => i.ToString()).ToArray();
File.WriteAllLines(SecurityMasterFilePath, the_array);
//Generated the SecurityMaster.csv file
}
catch (Exception ex)
{
lblMessage.AppendText(ex.StackTrace + Environment.NewLine);
}
}
}
public bool TryFetchFromUri(Uri uri, out string fileName)
{
fileName = null;
if (!Monitor.TryEnter(concurrentLock))
{
Logger.LogAs(this, LogLevel.Error, "Cannot perform concurrent downloads, aborting...");
return(false);
}
#if PLATFORM_WINDOWS
using (var locker = new WindowsFileLocker(GetCacheIndexLockLocation()))
#else
using (var locker = new PosixFileLocker(GetCacheIndexLockLocation()))
#endif
{
if (TryGetFromCache(uri, out fileName))
{
fetchedFiles.Add(fileName, uri);
Monitor.Exit(concurrentLock);
return(true);
}
using (client = new WebClient())
{
fileName = TemporaryFilesManager.Instance.GetTemporaryFile();
try
{
var attempts = 0;
var success = false;
do
{
var downloadProgressHandler = EmulationManager.Instance.ProgressMonitor.Start(GenerateProgressMessage(uri), false, true);
Logger.LogAs(this, LogLevel.Info, "Downloading {0}.", uri);
var now = CustomDateTime.Now;
var bytesDownloaded = 0L;
client.DownloadProgressChanged += (sender, e) =>
{
var newNow = CustomDateTime.Now;
var period = newNow - now;
if (period > progressUpdateThreshold)
{
downloadProgressHandler.UpdateProgress(e.ProgressPercentage,
GenerateProgressMessage(uri,
e.BytesReceived, e.TotalBytesToReceive, e.ProgressPercentage, 1.0 * (e.BytesReceived - bytesDownloaded) / period.TotalSeconds));
now = newNow;
bytesDownloaded = e.BytesReceived;
}
};
var wasCancelled = false;
Exception exception = null;
var resetEvent = new ManualResetEvent(false);
client.DownloadFileCompleted += delegate(object sender, AsyncCompletedEventArgs e)
{
exception = e.Error;
if (e.Cancelled)
{
wasCancelled = true;
}
resetEvent.Set();
downloadProgressHandler.Finish();
};
client.DownloadFileAsync(uri, fileName);
resetEvent.WaitOne();
downloadProgressHandler.Finish();
if (wasCancelled)
{
Logger.LogAs(this, LogLevel.Info, "Download cancelled.");
File.Delete(fileName);
return(false);
}
if (exception != null)
{
var webException = exception as WebException;
File.Delete(fileName);
Logger.LogAs(this, LogLevel.Error, "Failed to download from {0}, reason: {1}.", uri, webException != null ? ResolveWebException(webException) : exception.Message);
return(false);
}
Logger.LogAs(this, LogLevel.Info, "Download done.");
if (uri.ToString().EndsWith(".gz", StringComparison.InvariantCulture))
{
var decompressionProgressHandler = EmulationManager.Instance.ProgressMonitor.Start(string.Format("Decompressing {0}", uri), false, false);
Logger.LogAs(this, LogLevel.Info, "Decompressing {0}.", uri);
var decompressedFile = TemporaryFilesManager.Instance.GetTemporaryFile();
using (var gzipStream = new GZipStream(File.OpenRead(fileName), CompressionMode.Decompress))
using (var outputStream = File.OpenWrite(decompressedFile))
{
gzipStream.CopyTo(outputStream);
}
fileName = decompressedFile;
Logger.LogAs(this, LogLevel.Info, "Decompression done");
decompressionProgressHandler.Finish();
}
}while (!(success = UpdateInCache(uri, fileName)) && attempts++ < 2);
if (!success)
{
Logger.LogAs(this, LogLevel.Error, "Download failed {0} times, wrong checksum or size, aborting.", attempts);
File.Delete(fileName);
return(false);
}
fetchedFiles.Add(fileName, uri);
return(true);
}
finally
{
Monitor.Exit(concurrentLock);
}
}
}
}
private void GetSensorValuesFromSettings()
{
string name = new Identifier(Identifier, "values").ToString();
string s = _settings.GetValue(name, null);
if (!string.IsNullOrEmpty(s))
{
try
{
byte[] array = Convert.FromBase64String(s);
DateTime now = DateTime.UtcNow;
using (MemoryStream memoryStream = new MemoryStream(array))
{
using (GZipStream gZipStream = new GZipStream(memoryStream, CompressionMode.Decompress))
{
using (MemoryStream destination = new MemoryStream())
{
gZipStream.CopyTo(destination);
destination.Seek(0, SeekOrigin.Begin);
using (BinaryReader reader = new BinaryReader(destination))
{
try
{
long t = 0;
long readLen = reader.BaseStream.Length - reader.BaseStream.Position;
while (readLen > 0)
{
t += reader.ReadInt64();
DateTime time = DateTime.FromBinary(t);
if (time > now)
{
break;
}
float value = reader.ReadSingle();
AppendValue(value, time);
readLen = reader.BaseStream.Length - reader.BaseStream.Position;
}
}
catch (EndOfStreamException)
{ }
}
}
}
}
}
catch
{
// Ignored.
}
}
if (_values.Count > 0)
{
AppendValue(float.NaN, DateTime.UtcNow);
}
//remove the value string from the settings to reduce memory usage
_settings.Remove(name);
}
private void HandleRpcResult(ulong messageId, int sequence, BinaryReader messageReader)
{
uint code = messageReader.ReadUInt32();
ulong requestId = messageReader.ReadUInt64();
long convertedReqID = Convert.ToInt64(requestId);
Debug.WriteLine($"HandleRpcResult: requestId - {requestId},converted - {convertedReqID}, messageId - {messageId}");
if (!runningRequests.ContainsKey(convertedReqID))
{
Debug.WriteLine("request don't find");
return;
}
var requestInfo = runningRequests[convertedReqID];
var request = requestInfo.Item1;
request.ConfirmReceived = true;
Debug.WriteLine("try parse rpcResult");
uint innerCode = messageReader.ReadUInt32();
if (innerCode == 0x2144ca19)
{ // rpc_error
Debug.WriteLine("rpc error");
int errorCode = messageReader.ReadInt32();
string errorMessage = Serializers.String.read(messageReader);
request.OnError(errorCode, errorMessage);
requestInfo.Item2.SetResult(true);
}
else if (innerCode == 0x3072cfa1)
{
try
{
Debug.WriteLine("gzip");
// gzip_packed
byte[] packedData = Serializers.Bytes.read(messageReader);
using (var ms = new MemoryStream())
{
using (var packedStream = new MemoryStream(packedData, false))
using (var zipStream = new GZipStream(packedStream, CompressionMode.Decompress))
{
zipStream.CopyTo(ms);
ms.Position = 0;
}
using (var compressedReader = new BinaryReader(ms))
{
request.deserializeResponse(compressedReader);
requestInfo.Item2.SetResult(true);
}
}
}
catch (ZlibException ex)
{
}
}
else
{
Debug.WriteLine("some else");
messageReader.BaseStream.Position -= 4;
request.deserializeResponse(messageReader);
requestInfo.Item2.SetResult(true);
}
}
public async Task SetContent(HttpContent content)
{
if (content is null)
{
throw new ArgumentNullException(nameof(content));
}
if (content.Headers.ContentType.MediaType != null &&
(content.Headers.ContentType.MediaType.StartsWith("text/", StringComparison.OrdinalIgnoreCase) ||
content.Headers.ContentType.MediaType.EndsWith("/json", StringComparison.OrdinalIgnoreCase)))
{
string asString = null;
if (content.Headers.ContentEncoding.Any(e => OrdinalIgnoreCase.Equals(AcceptEncodingGzip, e)))
{
using (var stream = await content.ReadAsStreamAsync())
using (var inflate = new GZipStream(stream, CompressionMode.Decompress))
using (var reader = new StreamReader(inflate, Encoding.UTF8))
{
asString = reader.ReadToEnd();
}
}
else if (content.Headers.ContentEncoding.Any(e => OrdinalIgnoreCase.Equals(AcceptEncodingDeflate, e)))
{
using (var stream = await content.ReadAsStreamAsync())
using (var inflate = new DeflateStream(stream, CompressionMode.Decompress))
using (var reader = new StreamReader(inflate, Encoding.UTF8))
{
asString = reader.ReadToEnd();
}
}
else
{
asString = await content.ReadAsStringAsync();
}
lock (_syncpoint)
{
_mediaType = content.Headers.ContentType.MediaType;
_string = asString;
}
}
else
{
byte[] asBytes = null;
if (content.Headers.ContentEncoding.Any(e => OrdinalIgnoreCase.Equals(AcceptEncodingGzip, e)))
{
using (var stream = await content.ReadAsStreamAsync())
using (var inflate = new GZipStream(stream, CompressionMode.Decompress))
using (var memory = new MemoryStream())
{
inflate.CopyTo(memory);
asBytes = memory.ToArray();
}
}
else if (content.Headers.ContentEncoding.Any(e => OrdinalIgnoreCase.Equals(AcceptEncodingDeflate, e)))
{
using (var stream = await content.ReadAsStreamAsync())
using (var inflate = new DeflateStream(stream, CompressionMode.Decompress))
using (var memory = new MemoryStream())
{
inflate.CopyTo(memory);
asBytes = memory.ToArray();
}
}
else
{
asBytes = await content.ReadAsByteArrayAsync();
}
lock (_syncpoint)
{
_mediaType = content.Headers.ContentType.MediaType;
_byteArray = asBytes;
}
}
}
public static void DecompressGZip(this byte[] @this, Stream output)
{
using (var ms = new MemoryStream(@this))
using (var stream = new GZipStream(ms, CompressionMode.Decompress))
stream.CopyTo(output);
}
public PeRunner(PeRunnerOptions opt)
{
_nextStart = opt.StartAddress;
Initialize(_nextStart);
using (this.EnterExit())
{
// load any predefined exports
_psx = new Psx(this);
_exports.Add("libpsxscl.so", BizExvoker.GetExvoker(_psx, CallingConventionAdapters.Waterbox));
_emu = new Emu(this);
_exports.Add("libemuhost.so", BizExvoker.GetExvoker(_emu, CallingConventionAdapters.Waterbox));
_syscalls = new Syscalls(this);
_exports.Add("__syscalls", BizExvoker.GetExvoker(_syscalls, CallingConventionAdapters.Waterbox));
// load and connect all modules, starting with the executable
var todoModules = new Queue <string>();
todoModules.Enqueue(opt.Filename);
while (todoModules.Count > 0)
{
var moduleName = todoModules.Dequeue();
if (!_exports.ContainsKey(moduleName))
{
var path = Path.Combine(opt.Path, moduleName);
var gzpath = path + ".gz";
byte[] data;
if (File.Exists(gzpath))
{
using (var fs = new FileStream(gzpath, FileMode.Open, FileAccess.Read))
{
var tmp = new byte[4];
fs.Seek(-4, SeekOrigin.End);
fs.Read(tmp, 0, 4);
int size = BitConverter.ToInt32(tmp, 0);
data = new byte[size];
var ms = new MemoryStream(data);
fs.Seek(0, SeekOrigin.Begin);
using (var gs = new GZipStream(fs, CompressionMode.Decompress))
gs.CopyTo(ms);
}
}
else
{
data = File.ReadAllBytes(path);
}
var module = new PeWrapper(moduleName, data, _nextStart);
ComputeNextStart(module.Size);
AddMemoryBlock(module.Memory, moduleName);
_savestateComponents.Add(module);
_disposeList.Add(module);
_exports.Add(moduleName, module);
_modules.Add(module);
foreach (var name in module.ImportsByModule.Keys)
{
todoModules.Enqueue(name);
}
}
}
_libcpatch = new LibcPatch(this);
_exports["libc.so"] = new PatchImportResolver(_exports["libc.so"], BizExvoker.GetExvoker(_libcpatch, CallingConventionAdapters.Waterbox));
ConnectAllImports();
// load all heaps
_heap = CreateHeapHelper(opt.SbrkHeapSizeKB, "brk-heap", true);
_sealedheap = CreateHeapHelper(opt.SealedHeapSizeKB, "sealed-heap", true);
_invisibleheap = CreateHeapHelper(opt.InvisibleHeapSizeKB, "invisible-heap", false);
_plainheap = CreateHeapHelper(opt.PlainHeapSizeKB, "plain-heap", true);
if (opt.MmapHeapSizeKB != 0)
{
_mmapheap = new MapHeap(_nextStart, opt.MmapHeapSizeKB * 1024, "mmap-heap");
_mmapheap.Memory.Activate();
ComputeNextStart(opt.MmapHeapSizeKB * 1024);
AddMemoryBlock(_mmapheap.Memory, "mmap-heap");
_savestateComponents.Add(_mmapheap);
_disposeList.Add(_mmapheap);
}
_syscalls.Init();
Console.WriteLine("About to enter unmanaged code");
if (Win32Hacks.IsDebuggerReallyPresent() && !System.Diagnostics.Debugger.IsAttached)
{
// this means that GDB or another unconventional debugger is attached.
// if that's the case, and it's observing this core, it probably wants a break
System.Diagnostics.Debugger.Break();
}
// run unmanaged init code
var libcEnter = _exports["libc.so"].SafeResolve("__libc_entry_routine");
var psxInit = _exports["libpsxscl.so"].SafeResolve("__psx_init");
var del = (LibcEntryRoutineD)CallingConventionAdapters.Waterbox.GetDelegateForFunctionPointer(libcEnter, typeof(LibcEntryRoutineD));
// the current mmglue code doesn't use the main pointer at all, and this just returns
del(IntPtr.Zero, psxInit, 0);
foreach (var m in _modules)
{
m.RunGlobalCtors();
}
/*try
* {
* _modules[0].RunExeEntry();
* //throw new InvalidOperationException("main() returned!");
* }
* catch //(EndOfMainException)
* {
* }
* _modules[0].RunGlobalCtors();
* foreach (var m in _modules.Skip(1))
* {
* if (!m.RunDllEntry())
* throw new InvalidOperationException("DllMain() returned false");
* m.RunGlobalCtors();
* }*/
}
}
public void Deserialize(BinaryReader reader)
{
if (new string(reader.ReadChars(4)) != "FSOv")
{
return;
}
Version = reader.ReadInt32();
Compressed = reader.ReadBoolean();
var uReader = reader;
if (Compressed)
{
var length = reader.ReadInt32();
var cStream = new MemoryStream(reader.ReadBytes(length));
var zipStream = new GZipStream(cStream, CompressionMode.Decompress);
var decompStream = new MemoryStream();
zipStream.CopyTo(decompStream);
decompStream.Seek(0, SeekOrigin.Begin);
reader = new BinaryReader(decompStream);
cStream.Close();
zipStream.Close();
}
if (Version > 26)
{
TS1 = reader.ReadBoolean();
}
Context = new VMContextMarshal(Version);
Context.Deserialize(reader);
int ents = reader.ReadInt32();
Entities = new VMEntityMarshal[ents];
for (int i = 0; i < ents; i++)
{
var type = reader.ReadByte();
var ent = (type == 1) ? (VMEntityMarshal) new VMAvatarMarshal(Version, TS1) : new VMGameObjectMarshal(Version, TS1);
ent.Deserialize(reader);
Entities[i] = ent;
}
int thrN = reader.ReadInt32();
Threads = new VMThreadMarshal[thrN];
for (int i = 0; i < thrN; i++)
{
Threads[i] = new VMThreadMarshal(Version);
Threads[i].Deserialize(reader);
}
int mtgN = reader.ReadInt32();
MultitileGroups = new VMMultitileGroupMarshal[mtgN];
for (int i = 0; i < mtgN; i++)
{
MultitileGroups[i] = new VMMultitileGroupMarshal(Version);
MultitileGroups[i].Deserialize(reader);
}
int globs = reader.ReadInt32();
GlobalState = new short[globs];
for (int i = 0; i < globs; i++)
{
GlobalState[i] = reader.ReadInt16();
}
PlatformState = (TS1)?(VMAbstractLotState) new VMTS1LotState(Version):new VMTSOLotState(Version);
PlatformState.Deserialize(reader);
ObjectId = reader.ReadInt16();
if (Version > 23)
{
if (reader.ReadBoolean())
{
Tuning = new DynamicTuning(reader);
}
}
if (Compressed)
{
reader.BaseStream.Close();
}
}
private void readClient()
{
bool skipStreaming = false;
List <byte> skipBuffer = new List <byte>();
List <byte> buffer = new List <byte>();
while (null != _clientTcpClient &&
_clientTcpClient.Connected &&
null != _clientSslStream &&
null != _serverTcpClient &&
_serverTcpClient.Connected &&
null != _serverSslStream)
{
Thread.Sleep(1);
try
{
int one = _clientSslStream.ReadByte();
if (one >= 0)
{
if (!skipStreaming)
{
buffer.Add((byte)one);
_serverSslStream.WriteByte((byte)one);
_serverSslStream.Flush();
string content = Encoding.UTF8.GetString(buffer.ToArray());
if (content.ToUpper().EndsWith("HOST: "))
{
byte[] inject = Encoding.UTF8.GetBytes(string.Format("{0}\r\n", _Host));
buffer.AddRange(inject);
_serverSslStream.Write(inject.ToArray());
_serverSslStream.Flush();
skipStreaming = true;
skipBuffer.Clear();
//Console.WriteLine("Injected host");
}
/*
* else if (content.ToUpper().EndsWith("ACCEPT-ENCODING: "))
* {
* byte[] inject = Encoding.UTF8.GetBytes(string.Format("utf8\r\n"));
* buffer.AddRange(inject);
* _serverSslStream.Write(inject.ToArray());
* _serverSslStream.Flush();
* skipStreaming = true;
* skipBuffer.Clear();
* //Console.WriteLine("Injected encoding");
* }
*/
if (content.ToUpper().EndsWith("\r\n\r\n"))
{
Console.WriteLine("***readClient is being keptAlive***");
Console.WriteLine(content);
buffer.Clear();
}
else
{
//Console.WriteLine("Content Size: {0}", buffer.Count);
if (File.Exists("test.txt"))
{
using (
FileStream fs = File.Open("test.txt", FileMode.Truncate, FileAccess.Write,
FileShare.ReadWrite))
{
}
}
using (
FileStream fs = File.Open("test.txt", FileMode.OpenOrCreate, FileAccess.Write,
FileShare.ReadWrite))
{
byte[] contents = Encoding.UTF8.GetBytes("reading client...\r\n");
fs.Write(contents, 0, contents.Length);
fs.Flush();
contents = null;
try
{
using (MemoryStream ms = new MemoryStream(buffer.ToArray()))
{
using (GZipStream gzipStream = new GZipStream(ms, CompressionMode.Decompress))
{
gzipStream.CopyTo(fs);
gzipStream.Flush();
fs.Flush();
}
}
}
catch (Exception)
{
contents = buffer.ToArray();
fs.Write(contents, 0, contents.Length);
fs.Flush();
}
}
}
}
else
{
skipBuffer.Add((byte)one);
//Console.WriteLine("Skip Buffer={0}", Encoding.UTF8.GetString(skipBuffer.ToArray()));
if (Encoding.UTF8.GetString(skipBuffer.ToArray()).EndsWith("\r\n"))
{
//Console.WriteLine("Found end of line");
skipStreaming = false;
//Console.WriteLine("***Client read so far***");
//Console.WriteLine(Encoding.UTF8.GetString(buffer.ToArray()));
}
}
}
else if (buffer.Count > 0)
{
Console.WriteLine("***Client read***");
Console.WriteLine(Encoding.UTF8.GetString(buffer.ToArray()));
buffer.Clear();
}
}
catch (Exception e)
{
Console.Error.WriteLine("Failed to read from client exception={0}", e);
closeConnections();
return;
}
}
Console.WriteLine("readClient complete");
}
//
// Giai nen file TUDIEN duoc nen o dang .gz
//
public void GiaiNen(string DAUVAO, string DAURA)
{
using (FileStream streamMoFileGiaiNen = new FileStream(DAUVAO, FileMode.Open))
{
using (FileStream streamSauKhiGiaiNen = File.Create(DAURA))
{
using (GZipStream gzreader = new GZipStream(streamMoFileGiaiNen, CompressionMode.Decompress))
{
gzreader.CopyTo(streamSauKhiGiaiNen);
}
}
}
}
private bool HandleRpcResult(ulong messageId, int sequence, BinaryReader messageReader, TeleSharp.TL.TLMethod request)
{
uint code = messageReader.ReadUInt32();
ulong requestId = messageReader.ReadUInt64();
if (requestId == (ulong)request.MessageId)
{
request.ConfirmReceived = true;
}
//throw new NotImplementedException();
/*
* lock (runningRequests)
* {
* if (!runningRequests.ContainsKey(requestId))
* {
* logger.warning("rpc response on unknown request: {0}", requestId);
* messageReader.BaseStream.Position -= 12;
* return false;
* }
*
* request = runningRequests[requestId];
* runningRequests.Remove(requestId);
* }
*/
uint innerCode = messageReader.ReadUInt32();
if (innerCode == 0x2144ca19)
{ // rpc_error
int errorCode = messageReader.ReadInt32();
string errorMessage = Serializers.String.read(messageReader);
if (errorMessage.StartsWith("FLOOD_WAIT_"))
{
var resultString = Regex.Match(errorMessage, @"\d+").Value;
var seconds = int.Parse(resultString);
throw new FloodException(TimeSpan.FromSeconds(seconds));
}
else if (errorMessage.StartsWith("PHONE_MIGRATE_"))
{
var resultString = Regex.Match(errorMessage, @"\d+").Value;
var dcIdx = int.Parse(resultString);
throw new PhoneMigrationException(dcIdx);
}
else if (errorMessage.StartsWith("FILE_MIGRATE_"))
{
var resultString = Regex.Match(errorMessage, @"\d+").Value;
var dcIdx = int.Parse(resultString);
throw new FileMigrationException(dcIdx);
}
else if (errorMessage.StartsWith("USER_MIGRATE_"))
{
var resultString = Regex.Match(errorMessage, @"\d+").Value;
var dcIdx = int.Parse(resultString);
throw new UserMigrationException(dcIdx);
}
else if (errorMessage.StartsWith("NETWORK_MIGRATE_"))
{
var resultString = Regex.Match(errorMessage, @"\d+").Value;
var dcIdx = int.Parse(resultString);
throw new NetworkMigrationException(dcIdx);
}
else if (errorMessage == "PHONE_CODE_INVALID")
{
throw new InvalidPhoneCodeException("The numeric code used to authenticate does not match the numeric code sent by SMS/Telegram");
}
else if (errorMessage == "SESSION_PASSWORD_NEEDED")
{
throw new CloudPasswordNeededException("This Account has Cloud Password !");
}
else
{
throw new InvalidOperationException(errorMessage);
}
}
else if (innerCode == 0x3072cfa1)
{
try
{
// gzip_packed
byte[] packedData = Serializers.Bytes.read(messageReader);
using (var ms = new MemoryStream())
{
using (var packedStream = new MemoryStream(packedData, false))
using (var zipStream = new GZipStream(packedStream, CompressionMode.Decompress))
{
zipStream.CopyTo(ms);
ms.Position = 0;
}
using (var compressedReader = new BinaryReader(ms))
{
request.DeserializeResponse(compressedReader);
}
}
}
catch (ZlibException ex)
{
}
}
else
{
messageReader.BaseStream.Position -= 4;
request.DeserializeResponse(messageReader);
}
return(false);
}
/// <summary>
/// Reads an NBT file and puts it in an NBTFile object
/// </summary>
/// <param name="path">The path to read from</param>
/// <returns>The NBTFile object</returns>
public static NBTFile FromFile(string path)
{
using (FileStream f = new FileStream(path, FileMode.Open))
{
NBTFile ret = new NBTFile();
bool iscomp = true;
int _b = f.ReadByte();
if (_b == -1 || _b != 0x1f)
{
iscomp = false;
}
iscomp = iscomp ? f.ReadByte() == 0x8b : iscomp;
f.Seek(0L, SeekOrigin.Begin);
f.Seek(0L, SeekOrigin.Begin);
using (MemoryStream m = new MemoryStream((int)f.Length))
{
if (iscomp)
{
using (GZipStream g = new GZipStream(f, CompressionMode.Decompress, true)) g.CopyTo(m, (int)f.Length);
}
else
{
f.CopyTo(m);
}
m.Seek(1L, SeekOrigin.Begin);
using (BinaryDataReader b = new BinaryDataReader(m, Encoding.UTF8, true))
{
b.ByteOrder = ByteOrder.BigEndian;
ret = Read(b).ToFile();
}
}
return(ret);
}
}
public QRCodeData(byte[] rawData, Compression compressMode)
{
var bytes = new List <byte>(rawData);
//Decompress
if (compressMode.Equals(Compression.Deflate))
{
using (var input = new MemoryStream(bytes.ToArray()))
{
using (var output = new MemoryStream())
{
using (var dstream = new DeflateStream(input, CompressionMode.Decompress))
{
dstream.CopyTo(output);
}
bytes = new List <byte>(output.ToArray());
}
}
}
else if (compressMode.Equals(Compression.GZip))
{
using (var input = new MemoryStream(bytes.ToArray()))
{
using (var output = new MemoryStream())
{
using (var dstream = new GZipStream(input, CompressionMode.Decompress))
{
dstream.CopyTo(output);
}
bytes = new List <byte>(output.ToArray());
}
}
}
if (bytes[0] != 0x51 || bytes[1] != 0x52 || bytes[2] != 0x52)
{
throw new Exception("Invalid raw data file. Filetype doesn't match \"QRR\".");
}
//Set QR code version
var sideLen = (int)bytes[4];
bytes.RemoveRange(0, 5);
this.Version = (sideLen - 21 - 8) / 4 + 1;
//Unpack
var modules = new Queue <bool>();
foreach (var b in bytes)
{
var bArr = new BitArray(new byte[] { b });
for (int i = 7; i >= 0; i--)
{
modules.Enqueue((b & (1 << i)) != 0);
}
}
//Build module matrix
this.ModuleMatrix = new List <BitArray>();
for (int y = 0; y < sideLen; y++)
{
this.ModuleMatrix.Add(new BitArray(sideLen));
for (int x = 0; x < sideLen; x++)
{
this.ModuleMatrix[y][x] = modules.Dequeue();
}
}
}
/// <summary>
/// "transfer-encoding:chunked is not supported.
/// Workaround: Set 'git config --add --global http.postBuffer 10485760'
/// </summary>
/// <param name="serviceName"></param>
private void ServiceRpc(string serviceName)
{
context.Response.ContentType = string.Format("application/x-git-{0}-result", serviceName);
var fin = Path.GetTempFileName();
var fout = Path.GetTempFileName();
using (var file = File.Create(fin))
{
var encoding = context.Request.Headers["Content-Encoding"];
if (string.IsNullOrEmpty(encoding))
encoding = context.Request.ContentEncoding.EncodingName;
if (encoding.Equals("gzip"))
{
using (GZipStream decomp = new GZipStream(context.Request.InputStream, CompressionMode.Decompress))
{
decomp.CopyTo(file);
}
}
else
{
context.Request.InputStream.CopyTo(file);
}
}
Git.RunGitCmd(string.Format("{0} --stateless-rpc \"{1}\" < \"{2}\" > \"{3}\"", serviceName, gitWorkingDir, fin, fout));
context.Response.WriteFile(fout);
context.Response.End();
File.Delete(fin);
File.Delete(fout);
}
private void Open(Random random)
{
lock (this)
{
Stream @is;
bool needSkip = true, failed = false;
long size = 0L, seekTo = 0L;
try
{
// LUCENENET: We have embedded the default file, so if that filename is passed,
// open the local resource instead of an external file.
if (path == LuceneTestCase.DEFAULT_LINE_DOCS_FILE)
{
@is = this.GetType().getResourceAsStream(path);
}
else
{
@is = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
}
}
catch (Exception)
{
failed = true;
// if its not in classpath, we load it as absolute filesystem path (e.g. Hudson's home dir)
FileInfo file = new FileInfo(path);
size = file.Length;
if (path.EndsWith(".gz", StringComparison.Ordinal))
{
// if it is a gzip file, we need to use InputStream and slowly skipTo:
@is = new FileStream(file.FullName, FileMode.Append, FileAccess.Write, FileShare.Read);
}
else
{
// optimized seek using RandomAccessFile:
seekTo = RandomSeekPos(random, size);
FileStream channel = new FileStream(path, FileMode.Open);
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("TEST: LineFileDocs: file seek to fp=" + seekTo + " on open");
}
channel.Position = seekTo;
@is = new FileStream(channel.ToString(), FileMode.Append, FileAccess.Write, FileShare.Read);
needSkip = false;
}
}
if (!failed)
{
// if the file comes from Classpath:
size = @is.Length;// available();
}
if (path.EndsWith(".gz", StringComparison.Ordinal))
{
using (var gzs = new GZipStream(@is, CompressionMode.Decompress))
{
var temp = new MemoryStream();
gzs.CopyTo(temp);
// Free up the previous stream
@is.Dispose();
// Use the decompressed stream now
@is = temp;
}
// guestimate:
size = (long)(size * 2.8);
}
// If we only have an InputStream, we need to seek now,
// but this seek is a scan, so very inefficient!!!
if (needSkip)
{
seekTo = RandomSeekPos(random, size);
if (LuceneTestCase.VERBOSE)
{
Console.WriteLine("TEST: LineFileDocs: stream skip to fp=" + seekTo + " on open");
}
@is.Position = seekTo;
}
// if we seeked somewhere, read until newline char
if (seekTo > 0L)
{
int b;
byte[] bytes = new byte[sizeof(int)];
do
{
@is.Read(bytes, 0, sizeof(int));
b = BitConverter.ToInt32(bytes, 0);
} while (b >= 0 && b != 13 && b != 10);
}
//CharsetDecoder decoder = StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT).onUnmappableCharacter(CodingErrorAction.REPORT);
MemoryStream ms = new MemoryStream();
@is.CopyTo(ms);
reader = new StringReader(Encoding.UTF8.GetString(ms.ToArray()));//, BUFFER_SIZE);
if (seekTo > 0L)
{
// read one more line, to make sure we are not inside a Windows linebreak (\r\n):
reader.ReadLine();
}
@is.Dispose();
}
}
/// <summary>
/// Compresses the specified buffer stream using the G-Zip compression algorithm.
/// </summary>
/// <param name="buffer">The buffer.</param>
/// <param name="method">The method.</param>
/// <param name="mode">The mode.</param>
/// <returns>Block of bytes of compressed stream</returns>
public static MemoryStream Compress(
this Stream buffer,
CompressionMethod method = CompressionMethod.Gzip,
CompressionMode mode = CompressionMode.Compress)
{
buffer.Position = 0;
var targetStream = new MemoryStream();
switch (method)
{
case CompressionMethod.Deflate:
if (mode == CompressionMode.Compress)
{
using (var compressor = new DeflateStream(targetStream, CompressionMode.Compress, true))
{
buffer.CopyTo(compressor, 1024);
buffer.CopyTo(compressor);
// WebSocket use this
targetStream.Write(Last, 0, 1);
targetStream.Position = 0;
}
}
else
{
using (var compressor = new DeflateStream(buffer, CompressionMode.Decompress))
{
compressor.CopyTo(targetStream);
}
}
break;
case CompressionMethod.Gzip:
if (mode == CompressionMode.Compress)
{
using (var compressor = new GZipStream(targetStream, CompressionMode.Compress, true))
{
buffer.CopyTo(compressor);
}
}
else
{
using (var compressor = new GZipStream(buffer, CompressionMode.Decompress))
{
compressor.CopyTo(targetStream);
}
}
break;
case CompressionMethod.None:
buffer.CopyTo(targetStream);
break;
default:
throw new ArgumentOutOfRangeException(nameof(method), method, null);
}
return(targetStream);
}
public static bool FromAssemblyToFileByResourceName(
string byResourceName,
FileInfo toFile,
CompressionType compressionType)
{
if (string.IsNullOrWhiteSpace(byResourceName))
{
return(false);
}
if (toFile == null)
{
return(false);
}
try
{
var binaryDirectory = toFile.Directory;
if (binaryDirectory != null && !binaryDirectory.Exists)
{
binaryDirectory.Create();
}
var assembly = Assembly.GetCallingAssembly();
var doesResourceExist = false;
foreach (var name in assembly.GetManifestResourceNames())
{
if (byResourceName.Equals(name))
{
doesResourceExist = true;
}
}
if (!doesResourceExist)
{
Logger.GetInstance(typeof(Extract)).Error("Can not find resource \"" + byResourceName);
return(false);
}
using (var stream = assembly.GetManifestResourceStream(byResourceName))
{
if (stream == null)
{
return(false);
}
if (compressionType == CompressionType.Gzip)
{
using (var gZipStream = new GZipStream(stream, CompressionMode.Decompress))
{
using (var fileStream = toFile.OpenWrite())
{
gZipStream.CopyTo(fileStream);
}
}
}
else
{
using (var fileStream = toFile.OpenWrite())
{
stream.CopyTo(fileStream);
}
}
}
return(true);
}
catch (Exception e)
{
Logger.GetInstance(typeof(Extract)).Error("Can not extract resource \"" + byResourceName + "\" to \"" + toFile + "\": " + e.Message);
}
return(false);
}