public static void generateTwoBitDataAndSend(Stream outputStream, long dataSize) { int bufferSize = 512 * 1025; // 512 KB long chunkNum = dataSize / bufferSize; int extraBytes = (int)dataSize % bufferSize; List <byte> data = new List <byte>(); MemoryStream memStream = new MemoryStream(); GZipStream gzipStream = new GZipStream(memStream, CompressionMode.Compress, true); var compressedbuffer = new byte[1]; int bitIndex = 0; int oneByte = 0; char[] randomCharBuffer = new char[1]; initRandomCharBuffer(ref randomCharBuffer, ' '); for (int index = 0; index < chunkNum; index++) { for (int byteIndex = 0; byteIndex < bufferSize; byteIndex++) { int charIndex = rand.Next() % randomCharBuffer.Length; char oneChar = randomCharBuffer[charIndex]; oneByte = oneByte | FormatData.charCodeToByte(oneChar) << bitIndex; bitIndex += 2; if (bitIndex == 8) { data.Add((byte)oneByte); bitIndex = 0; oneByte = 0; } } byte[] buffer = data.ToArray(); data.Clear(); gzipStream.Write(buffer, 0, (int)buffer.Length); compressedbuffer = memStream.ToArray(); outputStream.Write(compressedbuffer, 0, compressedbuffer.Length); memStream.Seek(0, SeekOrigin.Begin); memStream.SetLength(0); memStream.Capacity = 10; } if (extraBytes > 0) { for (int byteIndex = 0; byteIndex < extraBytes; byteIndex++) { int charIndex = rand.Next() % randomCharBuffer.Length; char oneChar = randomCharBuffer[charIndex]; oneByte = oneByte | FormatData.charCodeToByte(oneChar) << bitIndex; bitIndex += 2; if (bitIndex == 8) { data.Add((byte)oneByte); bitIndex = 0; oneByte = 0; } } if (bitIndex > 0) { data.Add((byte)oneByte); } else { bitIndex = 8; } byte[] buffer = data.ToArray(); data.Clear(); gzipStream.Write(buffer, 0, (int)buffer.Length); writeBitsComplement(gzipStream, bitIndex); gzipStream.Flush(); gzipStream.Close(); compressedbuffer = memStream.ToArray(); outputStream.Write(compressedbuffer, 0, compressedbuffer.Length); } }
public static void generateTwoBitDataAndSend(Stream outputStream, long dataSize, bool compressed, double[] ratios = null) { int bufferSize = 1024 * 1024; // 1024 KB long chunkNum = dataSize / bufferSize; int extraBytes = (int)dataSize % bufferSize; List <byte> data = new List <byte>(); MemoryStream memStream = new MemoryStream(); GZipStream gzipStream = new GZipStream(memStream, CompressionMode.Compress, true); var databuffer = new byte[1]; int bitIndex = 0; int oneByte = 0; char[] randomCharBuffer = new char[1]; double[] randomCharRatios; if (ratios != null) { randomCharRatios = ratios; } else { randomCharRatios = randomRatios(specialChars.Length); } byte[] buffer; for (int index = 0; index < chunkNum; index++) { StringBuilder textData = new StringBuilder(); GenerateRandomeData(textData, bufferSize, randomCharRatios); randomCharBuffer = textData.ToString().ToCharArray(); for (int byteIndex = 0; byteIndex < bufferSize; byteIndex++) { char oneChar = randomCharBuffer[byteIndex]; oneByte = oneByte | FormatData.charCodeToByte(oneChar) << bitIndex; bitIndex += 2; if (bitIndex == 8) { data.Add((byte)oneByte); bitIndex = 0; oneByte = 0; } } buffer = data.ToArray(); data.Clear(); if (compressed) { gzipStream.Write(buffer, 0, (int)buffer.Length); } else { memStream.Write(buffer, 0, (int)buffer.Length); } databuffer = memStream.ToArray(); outputStream.Write(databuffer, 0, databuffer.Length); memStream.Seek(0, SeekOrigin.Begin); memStream.SetLength(0); memStream.Capacity = 10; } if (extraBytes > 0) { StringBuilder textData = new StringBuilder(); GenerateRandomeData(textData, extraBytes, randomCharRatios); randomCharBuffer = textData.ToString().ToCharArray(); for (int byteIndex = 0; byteIndex < extraBytes; byteIndex++) { char oneChar = randomCharBuffer[byteIndex]; oneByte = oneByte | FormatData.charCodeToByte(oneChar) << bitIndex; bitIndex += 2; if (bitIndex == 8) { data.Add((byte)oneByte); bitIndex = 0; oneByte = 0; } } } if (bitIndex > 0) { data.Add((byte)oneByte); } else { bitIndex = 8; } buffer = data.ToArray(); data.Clear(); if (compressed) { gzipStream.Write(buffer, 0, (int)buffer.Length); writeBitsComplement(gzipStream, bitIndex); } else { memStream.Write(buffer, 0, (int)buffer.Length); } long currentpos = memStream.Position; long length = memStream.Length; if (compressed) { gzipStream.Flush(); gzipStream.Close(); } databuffer = memStream.ToArray(); outputStream.Write(databuffer, 0, databuffer.Length); }
public override void handleGETRequest(HttpProcessor p) { bool shouldCompressData = false; if ((String)p.httpHeaders["Accept-Encoding"] == "gzip") { shouldCompressData = true; } if (p.http_url.Contains(".txt")) { string[] allTextFiles = getTextFiles(); string findFile = Path.GetFileName(p.http_url); bool precoded = false; encodingType dataEncoding = encodingType.Origin; if (findFile.Contains("Preconfig")) { findFile = findFile.Remove(0, "Preconfig".Length); precoded = true; } else { if (findFile.Contains("Raw")) { int fileTypeLoc = findFile.ToLower().IndexOf("raw"); findFile = findFile.Remove(fileTypeLoc, "Raw".Length); dataEncoding = encodingType.Origin; } } int strNumber = 0; int strIndex = -1; for (strNumber = 0; strNumber < allTextFiles.Length; strNumber++) { strIndex = allTextFiles[strNumber].ToLower().IndexOf(findFile.ToLower()); if (strIndex >= 0) { break; } } // do the server have the file if (strIndex < 0) { // the server cant find the required file p.writeFailure(); return; } var textFileName = allTextFiles[strNumber]; var outputFileName = textFileName; // if it was encoded and compressed, it will be ready to be send Stream fs = null; try { if (!precoded) { outputFileName = Path.GetTempFileName(); if (dataEncoding == encodingType.Origin) { outputFileName = textFileName; if (shouldCompressData) { fs = new FileStream(outputFileName, FileMode.Open, FileAccess.Read, FileShare.Read); FileInfo inputFileInfo = new FileInfo(outputFileName); p.writeSuccess("text/plain", 0, shouldCompressData, useStandardHeaders); p.outputStream.Flush(); FormatData.streamCopyToWithGzipCompression(fs, p.outputStream.BaseStream); p.outputStream.Flush(); p.outputStream.BaseStream.Flush(); } else { fs = File.Open(textFileName, FileMode.Open, FileAccess.Read, FileShare.Read); FileInfo fileInfo = new FileInfo(outputFileName); p.writeSuccess("text/plain", fileInfo.Length, shouldCompressData, useStandardHeaders); p.outputStream.Flush(); fs.CopyTo(p.outputStream.BaseStream); } } } else // precoded and compressed { fs = File.Open(outputFileName, FileMode.Open); FileInfo fileInfo = new FileInfo(outputFileName); p.writeSuccess("text/plain", fileInfo.Length, shouldCompressData, useStandardHeaders); p.outputStream.Flush(); fs.CopyTo(p.outputStream.BaseStream); } } catch (IOException ex) { Console.WriteLine("An error has occured {0}", ex.Message); } catch (Exception ex) { Console.WriteLine("An error has occured {0}", ex.Message); } finally { p.outputStream.BaseStream.Flush(); if (fs != null) { fs.Close(); } } } // if the client asks for autogenerated data else if (p.http_url.Contains("Auto")) { double[] ratios = null; if (p.httpHeaders["RatioType"].ToString() == "Manual") { ratios = new double[4]; ratios[0] = double.Parse(p.httpHeaders["A"].ToString()); ratios[1] = double.Parse(p.httpHeaders["C"].ToString()); ratios[2] = double.Parse(p.httpHeaders["G"].ToString()); ratios[3] = double.Parse(p.httpHeaders["T"].ToString()); } long dataSize = long.Parse(p.httpHeaders["DataSize"].ToString()); if (p.http_url.Contains("Raw")) { // auto data generation p.writeSuccess("text/plain", 0, shouldCompressData, useStandardHeaders); p.outputStream.Flush(); FormatData.generateRawDataAndSend(p.outputStream.BaseStream, dataSize, shouldCompressData, ratios); p.outputStream.BaseStream.Flush(); p.outputStream.Flush(); return; } else { p.writeFailure(); } } else { p.writeFailure(); } }