private void processImages()
{
AzureWrapper imageazure;
try
{
imageazure = new AzureWrapper(AzureStorage.VIDEO_CONTAINER_NAME);
}
catch (Exception ex)
{
throw ex;
}
while (true)
{
try
{
CloudQueueMessage message = imageazure.Queue.GetMessage();
string id = message.AsString;
IListBlobItem blobitem = imageazure.BlobContainer.ListBlobs().Where(b => b.Uri.ToString().Contains(id)).First();
CloudBlob blob = imageazure.BlobContainer.GetBlobReference(blobitem.Uri.ToString());
byte[] package = blob.DownloadByteArray();
Compression<ImageFrameSerialized> comp = new Compression<ImageFrameSerialized>();
ImageFrameSerialized frame = comp.GZipUncompress(package);
}
catch (Exception ex)
{
string message = ex.Message;
}
}
}
public void CanDecompressFile()
{
var compression = new Compression();
string value = compression.Decompress(@"Resources\commit");
// should do better than this!
Assert.That(value, Is.Not.Null);
}
protected override void Kinect_NewRecordedAudio(byte[] audio)
{
base.Kinect_NewRecordedAudio(audio);
if (NewCompressedAudioStream != null)
{
Compression<byte[]> comp = new Compression<byte[]>();
byte[] compressedValue = comp.GzipCompress(audio);
NewCompressedAudioStream(compressedValue, audio);
}
}
protected override void Kinect_NewDepthFrame(Depth depthImage)
{
base.Kinect_NewDepthFrame(depthImage);
if (NewCompressedKinectDepthFrame != null)
{
Compression<DepthImageFrame> comp = new Compression<DepthImageFrame>();
byte[] compressedVal = comp.GzipCompress(depthImage.DepthFrame);
NewCompressedKinectDepthFrame(compressedVal, depthImage.DepthFrame);
}
}
public CompressionOption ConvertCompIntoCompressOption(Compression comp)
{
CompressionOption compress = CompressionOption.Normal;
switch (comp)
{
case Compression.Normal: compress = CompressionOption.Normal; break;
case Compression.High: compress = CompressionOption.Maximum; break;
case Compression.Fast: compress = CompressionOption.Fast; break;
}
return compress;
}
protected override void Kinect_NewSkeletonFrame(Skel skeletonImage)
{
base.Kinect_NewSkeletonFrame(skeletonImage);
if (NewCompressedSkeletonFrame != null)
{
Compression<SkeletonFrame> comp = new Compression<SkeletonFrame>();
byte[] compressedVal = comp.GzipCompress(skeletonImage.Skeleton);
NewCompressedSkeletonFrame(compressedVal, skeletonImage.Skeleton);
}
}
public static IIndexSerializer Create(Compression compression)
{
switch (compression)
{
case Compression.No:
return new SimpleIndexSerializer();
case Compression.Yes:
return new CompressingIndexSerializer(new StreamFactory(), new VariableByteNumberEncoder(),
new NumberLengthReducer());
default:
throw new ArgumentOutOfRangeException(nameof(compression), compression, "Unknown compression type");
}
}
public static void WriteToDisk(ushort[][] data, String fileName, int imageWidth, int imageHeight, int samplesPerPixel = 1, Compression compression = Compression.NONE, int bitsPerSample = 16)
{
if (data == null)
throw new Exception("no data provided");
using (Tiff imagesData = Tiff.Open(fileName, "w"))
{
for (uint page = 0; page < data.Length; page++)
{
imagesData.SetField(TiffTag.IMAGEWIDTH, imageWidth.ToString(CultureInfo.InvariantCulture));
imagesData.SetField(TiffTag.IMAGELENGTH, imageHeight.ToString(CultureInfo.InvariantCulture));
imagesData.SetField(TiffTag.COMPRESSION, compression);
imagesData.SetField(TiffTag.BITSPERSAMPLE, bitsPerSample.ToString(CultureInfo.InvariantCulture));
// imageData.SetField(TiffTag.SAMPLESPERPIXEL, samplesPerPixel);
imagesData.SetField(TiffTag.XRESOLUTION, 1);
imagesData.SetField(TiffTag.YRESOLUTION, 1);
imagesData.SetField(TiffTag.DATETIME, DateTime.Now);
// imageData.SetField(TiffTag.ROWSPERSTRIP, imageHeight.ToString(CultureInfo.InvariantCulture));
imagesData.SetField(TiffTag.PLANARCONFIG, PlanarConfig.CONTIG);
imagesData.SetField(TiffTag.PHOTOMETRIC, Photometric.MINISBLACK);
imagesData.SetField(TiffTag.FILLORDER, FillOrder.MSB2LSB);
imagesData.SetField(TiffTag.ORIENTATION, Orientation.TOPLEFT);
imagesData.SetField(TiffTag.RESOLUTIONUNIT, ResUnit.CENTIMETER);
imagesData.SetField(TiffTag.ARTIST, "ProjectStorm");
imagesData.SetField(TiffTag.IMAGEDESCRIPTION, "Test data constructed by openCL kernel of project storm");
// specify that it's a page within the multipage file
imagesData.SetField(TiffTag.SUBFILETYPE, FileType.PAGE);
// specify the page number
imagesData.SetField(TiffTag.PAGENUMBER, page, data.Length);
for (int i = 0; i < imageHeight; i++)
{
Byte[] buffer = new byte[data[page].Length * sizeof(ushort)];
Buffer.BlockCopy(data, i * imageWidth, buffer, 0, buffer.Length);
imagesData.WriteScanline(buffer, i);
}
imagesData.WriteDirectory();
}
imagesData.FlushData();
}
}
public Album RetrieveAlbumByName(IDBClient db, string albumName, Compression compression)
{
var db4oClient = db as Db4oClient;
// search for the album
if (db4oClient != null)
{
IEnumerable<Album> result = from Album a in db4oClient.Client
where a.Title == albumName &&
a.Compression.Equals(compression)
select a;
// return the first one if anything returned
if (result.Count() > 0)
{
return result.ToArray()[0];
}
}
return null;
}
public static ServerPacket CompressPacket(Packet packet)
{
// We need the opcode as uint.
var msg = BitConverter.GetBytes((uint)packet.Header.Message);
packet.Data[0] = msg[0];
packet.Data[1] = msg[1];
packet.Data[2] = msg[2];
packet.Data[3] = msg[3];
var compression = new Compression
{
UncompressedSize = packet.Data.Length,
UncompressedAdler = Helper.Adler32(packet.Data)
};
// Compress.
using (var ms = new MemoryStream())
{
using (var ds = new DeflateStream(ms, CompressionLevel.Fastest))
{
ds.Write(packet.Data, 0, packet.Data.Length);
ds.Flush();
}
compression.CompressedData = ms.ToArray();
}
compression.CompressedData[0] -= 1;
if ((compression.CompressedData[compression.CompressedData.Length - 1] & 8) == 8)
compression.CompressedData = compression.CompressedData.Combine(new byte[1]);
compression.CompressedData = compression.CompressedData.Combine(new byte[] { 0x00, 0x00, 0xFF, 0xFF });
compression.CompressedAdler = Helper.Adler32(compression.CompressedData);
return compression;
}
public CqlResult execute_cql_query(byte[] query, Compression compression)
{
send_execute_cql_query(query, compression);
return recv_execute_cql_query();
}
public void send_prepare_cql3_query(byte[] query, Compression compression)
#endif
{
oprot_.WriteMessageBegin(new TMessage("prepare_cql3_query", TMessageType.Call, seqid_));
prepare_cql3_query_args args = new prepare_cql3_query_args();
args.Query = query;
args.Compression = compression;
args.Write(oprot_);
oprot_.WriteMessageEnd();
#if SILVERLIGHT
return oprot_.Transport.BeginFlush(callback, state);
#else
oprot_.Transport.Flush();
#endif
}
public bool ProcessCompressOptions(string opt)
{
if (opt == "none")
{
m_defcompression = Compression.NONE;
}
else if (opt == "packbits")
{
m_defcompression = Compression.PACKBITS;
}
else if (opt.StartsWith("jpeg"))
{
m_defcompression = Compression.JPEG;
string[] options = opt.Split(new char[] { ':' });
for (int i = 1; i < options.Length; i++)
{
if (char.IsDigit(options[i][0]))
m_quality = int.Parse(options[i], CultureInfo.InvariantCulture);
else if (options[i] == "r")
m_jpegcolormode = JpegColorMode.RAW;
else
return false;
}
}
else if (opt.StartsWith("g3"))
{
if (!processG3Options(opt))
return false;
m_defcompression = Compression.CCITTFAX3;
}
else if (opt == "g4")
{
m_defcompression = Compression.CCITTFAX4;
}
else if (opt.StartsWith("lzw"))
{
int n = opt.IndexOf(':');
if (n != -1 && n < (opt.Length - 1))
m_defpredictor = short.Parse(opt.Substring(n + 1));
m_defcompression = Compression.LZW;
}
else if (opt.StartsWith("zip"))
{
int n = opt.IndexOf(':');
if (n != -1 && n < (opt.Length - 1))
m_defpredictor = short.Parse(opt.Substring(n + 1));
m_defcompression = Compression.ADOBE_DEFLATE;
}
else
return false;
return true;
}
/// <summary>
/// Prepare a CQL3 (Cassandra Query Language) statement by compiling and returning
/// - the type of CQL statement
/// - an id token of the compiled CQL stored on the server side.
/// - a count of the discovered bound markers in the statement
/// </summary>
/// <param name="query"></param>
/// <param name="compression"></param>
public CqlPreparedResult prepare_cql3_query(byte[] query, Compression compression)
{
#if !SILVERLIGHT
send_prepare_cql3_query(query, compression);
return recv_prepare_cql3_query();
#else
var asyncResult = Begin_prepare_cql3_query(null, null, query, compression);
return End_prepare_cql3_query(asyncResult);
#endif
}
public IAsyncResult send_execute_cql3_query(AsyncCallback callback, object state, byte[] query, Compression compression, ConsistencyLevel consistency)
public IAsyncResult Begin_execute_cql3_query(AsyncCallback callback, object state, byte[] query, Compression compression, ConsistencyLevel consistency)
{
return send_execute_cql3_query(callback, state, query, compression, consistency);
}
public CodecWithPredictor(Tiff tif, Compression scheme, string name)
: base(tif, scheme, name)
{
m_tagMethods = new CodecWithPredictorTagMethods();
}
/// <summary>
/// Initializes a new instance of the <see cref="TiffCodec"/> class.
/// </summary>
/// <param name="tif">An instance of <see cref="Tiff"/> class.</param>
/// <param name="scheme">The compression scheme for the codec.</param>
/// <param name="name">The name of the codec.</param>
public TiffCodec(Tiff tif, Compression scheme, string name)
{
m_scheme = scheme;
m_tif = tif;
m_name = name;
}
public OJpegCodec(Tiff tif, Compression scheme, string name)
: base(tif, scheme, name)
{
m_tagMethods = new OJpegCodecTagMethods();
}
/// <summary>
/// 比较当前记录是不是与硬盘上的记录相等
/// </summary>
/// <param name="page">页对像实体</param>
/// <param name="path">文件所在的路径</param>
/// <returns></returns>
public static bool IsTabPageEntitySync(this TabPageEntity page)
{
//如果文件不存在直接返回 false
if (!System.IO.File.Exists(page.GetRecordPath(true)))
{
return(false);
}
//实始化buff
byte[] bytearr = new byte[10];
//锁住读写文件时使用
lock (page)
{
//读取缓存文件内容
//如果多线程操作可能会引发文件访问异常,这里需要对,访问对象进行控制
bytearr = System.IO.File.ReadAllBytes(page.GetRecordPath(true));
}
//得到Zip文件解压缩后的数据
bytearr = Compression.DeCompress(bytearr);
//得到需要序列化的字符串
string Text = System.Text.UnicodeEncoding.Unicode.GetString(bytearr);
//创建一个序列化对像
System.Xml.Serialization.XmlSerializer xmlSerializer = new System.Xml.Serialization.XmlSerializer(typeof(TabPageEntity));
//得到文中得到的Page 对象
TabPageEntity diskPage = (TabPageEntity)xmlSerializer.Deserialize(new System.IO.StringReader(Text));
if (
diskPage.Book.Record.GUID == page.Book.Record.GUID
&&
diskPage.Book.Record.ID == page.Book.Record.ID
&&
diskPage.Book.Record.创建时间 == page.Book.Record.创建时间
&&
diskPage.Book.Record.分类标识 == page.Book.Record.分类标识
&&
diskPage.Book.Record.分类表ID == page.Book.Record.分类表ID
&&
diskPage.Book.Record.书名 == page.Book.Record.书名
&&
diskPage.Book.Record.说明 == page.Book.Record.说明
&&
diskPage.Book.Record.完本 == page.Book.Record.完本
&&
diskPage.Book.Record.周点击 == page.Book.Record.周鲜花
&&
diskPage.Book.Record.周鲜花 == page.Book.Record.周鲜花
&&
diskPage.Book.Record.总点击 == page.Book.Record.总点击
&&
diskPage.Book.Record.总鲜花 == page.Book.Record.总鲜花
&&
diskPage.Book.Record.最后更新时间 == page.Book.Record.最后更新时间
//分类比较结束 开始内容比较
&&
diskPage.Record.ID == page.Record.ID
&&
diskPage.Record.本记录GUID == page.Record.本记录GUID
&&
diskPage.Record.最后访问时间 == page.Record.最后访问时间
)
{
return(true);
}
else
{
return(false);
}
}
/// <summary>
/// Retrieves the codec registered for the specified compression scheme.
/// </summary>
/// <param name="scheme">The compression scheme.</param>
/// <returns>The codec registered for the specified compression scheme or <c>null</c>
/// if there is no codec registered for the given scheme.</returns>
/// <remarks>
/// <para>
/// LibTiff.Net supports a variety of compression schemes implemented by software codecs.
/// Each codec adheres to a modular interface that provides for the decoding and encoding
/// of image data; as well as some other methods for initialization, setup, cleanup, and
/// the control of default strip and tile sizes. Codecs are identified by the associated
/// value of the <see cref="TiffTag"/>.Compression tag.
/// </para>
/// <para>
/// Other compression schemes may be registered. Registered schemes can also override the
/// built-in versions provided by the library.
/// </para>
/// </remarks>
public TiffCodec FindCodec(Compression scheme)
{
for (codecList list = m_registeredCodecs; list != null; list = list.next)
{
if (list.codec.m_scheme == scheme)
return list.codec;
}
for (int i = 0; m_builtInCodecs[i] != null; i++)
{
TiffCodec codec = m_builtInCodecs[i];
if (codec.m_scheme == scheme)
return codec;
}
return null;
}
private void listView1_ItemActivate(object sender, EventArgs e)
{
int num = (int)new MKDSEditor(NARC.Unpack(Compression.LZ77Decompress(this.Rom.Root.GetFileByPath("\\data\\Course\\" + this.listView1.SelectedItems[0].Text + ".carc").Content)), NARC.Unpack(Compression.LZ77Decompress(this.Rom.Root.GetFileByPath("\\data\\Course\\" + this.listView1.SelectedItems[0].Text + "Tex.carc").Content)), this.Rom.Root).ShowDialog();
}
public byte[] GetRawData(Compression compressMode)
{
var bytes = new List <byte>();
//Add header - signature ("QRR")
bytes.AddRange(new byte[] { 0x51, 0x52, 0x52, 0x00 });
//Add header - rowsize
bytes.Add((byte)ModuleMatrix.Count);
//Build data queue
var dataQueue = new Queue <int>();
foreach (var row in ModuleMatrix)
{
foreach (var module in row)
{
dataQueue.Enqueue((bool)module ? 1 : 0);
}
}
for (int i = 0; i < 8 - (ModuleMatrix.Count * ModuleMatrix.Count) % 8; i++)
{
dataQueue.Enqueue(0);
}
//Process queue
while (dataQueue.Count > 0)
{
byte b = 0;
for (int i = 7; i >= 0; i--)
{
b += (byte)(dataQueue.Dequeue() << i);
}
bytes.Add(b);
}
var rawData = bytes.ToArray();
//Compress stream (optional)
if (compressMode.Equals(Compression.Deflate))
{
using (var output = new MemoryStream())
{
using (var dstream = new DeflateStream(output, CompressionMode.Compress))
{
dstream.Write(rawData, 0, rawData.Length);
}
rawData = output.ToArray();
}
}
else if (compressMode.Equals(Compression.GZip))
{
using (var output = new MemoryStream())
{
using (GZipStream gzipStream = new GZipStream(output, CompressionMode.Compress, true))
{
gzipStream.Write(rawData, 0, rawData.Length);
}
rawData = output.ToArray();
}
}
return(rawData);
}
private static void OnCompressedGump(PacketReader reader)
{
int senderSerial = reader.ReadInt32();
uint gumpId = reader.ReadUInt32();
Engine.GumpList.AddOrUpdate(gumpId, senderSerial, (k, v) => senderSerial);
int x = reader.ReadInt32();
int y = reader.ReadInt32();
int compressedLength = reader.ReadInt32();
int decompressedLength = reader.ReadInt32() + 1;
if (compressedLength <= 4)
{
return;
}
compressedLength -= 4;
byte[] decompressedBuffer = new byte[decompressedLength];
byte[] compressedBuffer = reader.ReadByteArray(compressedLength);
bool success = Compression.Uncompress(ref decompressedBuffer, ref decompressedLength, compressedBuffer,
compressedLength);
if (!success)
{
return;
}
string layout = Encoding.ASCII.GetString(decompressedBuffer);
int linesCount = reader.ReadInt32();
compressedLength = reader.ReadInt32();
decompressedLength = reader.ReadInt32() + 1;
string[] text = new string[linesCount];
if (compressedLength > 4)
{
compressedLength -= 4;
decompressedBuffer = new byte[decompressedLength];
compressedBuffer = reader.ReadByteArray(compressedLength);
success = Compression.Uncompress(ref decompressedBuffer, ref decompressedLength, compressedBuffer,
compressedLength);
if (!success)
{
return;
}
int offset = 0;
for (int i = 0; i < linesCount; i++)
{
int length = ((decompressedBuffer[offset] << 8) | decompressedBuffer[offset + 1]) * 2;
offset += 2;
text[i] = Encoding.BigEndianUnicode.GetString(decompressedBuffer, offset, length);
offset += length;
}
}
try
{
Gump gump = GumpParser.Parse(senderSerial, gumpId, x, y, layout, text);
Engine.Gumps.Add(gump);
GumpEvent?.Invoke(gumpId, senderSerial, gump);
}
catch (Exception e)
{
e.ToExceptionless().SetProperty("Serial", senderSerial).SetProperty("GumpID", gumpId)
.SetProperty("Layout", layout).SetProperty("Text", text)
.SetProperty("Packet", reader.GetData()).SetProperty("Player", Engine.Player.ToString())
.SetProperty("WorldItemCount", Engine.Items.Count())
.SetProperty("WorldMobileCount", Engine.Mobiles.Count()).Submit();
}
}
/// <summary>
/// Executes a CQL3 (Cassandra Query Language) statement and returns a
/// CqlResult containing the results.
/// </summary>
/// <param name="query"></param>
/// <param name="compression"></param>
/// <param name="consistency"></param>
public CqlResult execute_cql3_query(byte[] query, Compression compression, ConsistencyLevel consistency)
{
#if !SILVERLIGHT
send_execute_cql3_query(query, compression, consistency);
return recv_execute_cql3_query();
#else
var asyncResult = Begin_execute_cql3_query(null, null, query, compression, consistency);
return End_execute_cql3_query(asyncResult);
#endif
}
static Source <ArraySegment <byte>, AkkaNet.NotUsed> NonGroupingGzipMapper(Source <ArraySegment <byte>, AkkaNet.NotUsed> s) => s.Select(m => new ArraySegment <byte>(Compression.CompressToGzip(m.ToArray())));
public IAsyncResult Begin_prepare_cql3_query(AsyncCallback callback, object state, byte[] query, Compression compression)
{
return send_prepare_cql3_query(callback, state, query, compression);
}
static Source <ArraySegment <byte>, AkkaNet.NotUsed> GroupingGzipMapper(Source <ArraySegment <byte>, AkkaNet.NotUsed> s, LogsOutputGroupingConfig outputGroupingConfig) => s.GroupedWithin(outputGroupingConfig.MaxFrames, outputGroupingConfig.MaxDuration) .Select(b => new ArraySegment <byte>(Compression.CompressToGzip(b.SelectMany(a => a).ToArray())));
public IAsyncResult send_prepare_cql3_query(AsyncCallback callback, object state, byte[] query, Compression compression)
private static extern IntPtr ColumnChunkMetaData_Compression(IntPtr columnChunkMetaData, out Compression compression);
public bool Copy(Tiff inImage, Tiff outImage)
{
int width = 0;
FieldValue[] result = inImage.GetField(TiffTag.IMAGEWIDTH);
if (result != null)
{
width = result[0].ToInt();
outImage.SetField(TiffTag.IMAGEWIDTH, width);
}
int length = 0;
result = inImage.GetField(TiffTag.IMAGELENGTH);
if (result != null)
{
length = result[0].ToInt();
outImage.SetField(TiffTag.IMAGELENGTH, length);
}
short bitspersample = 1;
result = inImage.GetField(TiffTag.BITSPERSAMPLE);
if (result != null)
{
bitspersample = result[0].ToShort();
outImage.SetField(TiffTag.BITSPERSAMPLE, bitspersample);
}
short samplesperpixel = 1;
result = inImage.GetField(TiffTag.SAMPLESPERPIXEL);
if (result != null)
{
samplesperpixel = result[0].ToShort();
outImage.SetField(TiffTag.SAMPLESPERPIXEL, samplesperpixel);
}
if (m_compression != (Compression)(-1))
outImage.SetField(TiffTag.COMPRESSION, m_compression);
else
{
result = inImage.GetField(TiffTag.COMPRESSION);
if (result != null)
{
m_compression = (Compression)result[0].ToInt();
outImage.SetField(TiffTag.COMPRESSION, m_compression);
}
}
result = inImage.GetFieldDefaulted(TiffTag.COMPRESSION);
Compression input_compression = (Compression)result[0].ToInt();
result = inImage.GetFieldDefaulted(TiffTag.PHOTOMETRIC);
Photometric input_photometric = (Photometric)result[0].ToShort();
if (input_compression == Compression.JPEG)
{
/* Force conversion to RGB */
inImage.SetField(TiffTag.JPEGCOLORMODE, JpegColorMode.RGB);
}
else if (input_photometric == Photometric.YCBCR)
{
/* Otherwise, can't handle subsampled input */
result = inImage.GetFieldDefaulted(TiffTag.YCBCRSUBSAMPLING);
short subsamplinghor = result[0].ToShort();
short subsamplingver = result[1].ToShort();
if (subsamplinghor != 1 || subsamplingver != 1)
{
Console.Error.WriteLine("tiffcp: {0}: Can't copy/convert subsampled image.", inImage.FileName());
return false;
}
}
if (m_compression == Compression.JPEG)
{
if (input_photometric == Photometric.RGB && m_jpegcolormode == JpegColorMode.RGB)
outImage.SetField(TiffTag.PHOTOMETRIC, Photometric.YCBCR);
else
outImage.SetField(TiffTag.PHOTOMETRIC, input_photometric);
}
else if (m_compression == Compression.SGILOG || m_compression == Compression.SGILOG24)
{
outImage.SetField(TiffTag.PHOTOMETRIC, samplesperpixel == 1 ? Photometric.LOGL : Photometric.LOGLUV);
}
else
{
if (input_compression != Compression.JPEG)
copyTag(inImage, outImage, TiffTag.PHOTOMETRIC, 1, TiffType.SHORT);
}
if (m_fillorder != 0)
outImage.SetField(TiffTag.FILLORDER, m_fillorder);
else
copyTag(inImage, outImage, TiffTag.FILLORDER, 1, TiffType.SHORT);
/*
* Will copy `Orientation' tag from input image
*/
result = inImage.GetFieldDefaulted(TiffTag.ORIENTATION);
m_orientation = (Orientation)result[0].ToByte();
switch (m_orientation)
{
case Orientation.BOTRIGHT:
case Orientation.RIGHTBOT:
Tiff.Warning(inImage.FileName(), "using bottom-left orientation");
m_orientation = Orientation.BOTLEFT;
break;
case Orientation.LEFTBOT:
case Orientation.BOTLEFT:
break;
case Orientation.TOPRIGHT:
case Orientation.RIGHTTOP:
default:
Tiff.Warning(inImage.FileName(), "using top-left orientation");
m_orientation = Orientation.TOPLEFT;
break;
case Orientation.LEFTTOP:
case Orientation.TOPLEFT:
break;
}
outImage.SetField(TiffTag.ORIENTATION, m_orientation);
/*
* Choose tiles/strip for the output image according to
* the command line arguments (-tiles, -strips) and the
* structure of the input image.
*/
if (m_outtiled == -1)
{
if (inImage.IsTiled())
m_outtiled = 1;
else
m_outtiled = 0;
}
if (m_outtiled != 0)
{
/*
* Setup output file's tile width&height. If either
* is not specified, use either the value from the
* input image or, if nothing is defined, use the
* library default.
*/
if (m_tilewidth == -1)
{
result = inImage.GetFieldDefaulted(TiffTag.TILEWIDTH);
if (result != null)
m_tilewidth = result[0].ToInt();
}
if (m_tilelength == -1)
{
result = inImage.GetFieldDefaulted(TiffTag.TILELENGTH);
if (result != null)
m_tilelength = result[0].ToInt();
}
outImage.DefaultTileSize(ref m_tilewidth, ref m_tilelength);
outImage.SetField(TiffTag.TILEWIDTH, m_tilewidth);
outImage.SetField(TiffTag.TILELENGTH, m_tilelength);
}
else
{
/*
* RowsPerStrip is left unspecified: use either the
* value from the input image or, if nothing is defined,
* use the library default.
*/
if (m_rowsperstrip == 0)
{
result = inImage.GetField(TiffTag.ROWSPERSTRIP);
if (result == null)
m_rowsperstrip = outImage.DefaultStripSize(m_rowsperstrip);
else
m_rowsperstrip = result[0].ToInt();
if (m_rowsperstrip > length && m_rowsperstrip != -1)
m_rowsperstrip = length;
}
else if (m_rowsperstrip == -1)
m_rowsperstrip = length;
outImage.SetField(TiffTag.ROWSPERSTRIP, m_rowsperstrip);
}
if (m_config != PlanarConfig.UNKNOWN)
outImage.SetField(TiffTag.PLANARCONFIG, m_config);
else
{
result = inImage.GetField(TiffTag.PLANARCONFIG);
if (result != null)
{
m_config = (PlanarConfig)result[0].ToShort();
outImage.SetField(TiffTag.PLANARCONFIG, m_config);
}
}
if (samplesperpixel <= 4)
copyTag(inImage, outImage, TiffTag.TRANSFERFUNCTION, 4, TiffType.SHORT);
copyTag(inImage, outImage, TiffTag.COLORMAP, 4, TiffType.SHORT);
/* SMinSampleValue & SMaxSampleValue */
switch (m_compression)
{
case Compression.JPEG:
outImage.SetField(TiffTag.JPEGQUALITY, m_quality);
outImage.SetField(TiffTag.JPEGCOLORMODE, m_jpegcolormode);
break;
case Compression.LZW:
case Compression.ADOBE_DEFLATE:
case Compression.DEFLATE:
if (m_predictor != -1)
outImage.SetField(TiffTag.PREDICTOR, m_predictor);
else
{
result = inImage.GetField(TiffTag.PREDICTOR);
if (result != null)
{
m_predictor = result[0].ToShort();
outImage.SetField(TiffTag.PREDICTOR, m_predictor);
}
}
break;
case Compression.CCITTFAX3:
case Compression.CCITTFAX4:
if (m_compression == Compression.CCITTFAX3)
{
if (m_g3opts != Group3Opt.UNKNOWN)
outImage.SetField(TiffTag.GROUP3OPTIONS, m_g3opts);
else
{
result = inImage.GetField(TiffTag.GROUP3OPTIONS);
if (result != null)
{
m_g3opts = (Group3Opt)result[0].ToShort();
outImage.SetField(TiffTag.GROUP3OPTIONS, m_g3opts);
}
}
}
else
copyTag(inImage, outImage, TiffTag.GROUP4OPTIONS, 1, TiffType.LONG);
copyTag(inImage, outImage, TiffTag.BADFAXLINES, 1, TiffType.LONG);
copyTag(inImage, outImage, TiffTag.CLEANFAXDATA, 1, TiffType.LONG);
copyTag(inImage, outImage, TiffTag.CONSECUTIVEBADFAXLINES, 1, TiffType.LONG);
copyTag(inImage, outImage, TiffTag.FAXRECVPARAMS, 1, TiffType.LONG);
copyTag(inImage, outImage, TiffTag.FAXRECVTIME, 1, TiffType.LONG);
copyTag(inImage, outImage, TiffTag.FAXSUBADDRESS, 1, TiffType.ASCII);
break;
}
result = inImage.GetField(TiffTag.ICCPROFILE);
if (result != null)
outImage.SetField(TiffTag.ICCPROFILE, result[0], result[1]);
result = inImage.GetField(TiffTag.NUMBEROFINKS);
if (result != null)
{
short ninks = result[0].ToShort();
outImage.SetField(TiffTag.NUMBEROFINKS, ninks);
result = inImage.GetField(TiffTag.INKNAMES);
if (result != null)
{
string inknames = result[0].ToString();
string[] parts = inknames.Split(new char[] { '\0' });
int inknameslen = 0;
foreach (string part in parts)
inknameslen += part.Length + 1;
outImage.SetField(TiffTag.INKNAMES, inknameslen, inknames);
}
}
result = inImage.GetField(TiffTag.PAGENUMBER);
if (m_pageInSeq == 1)
{
if (m_pageNum < 0)
{
/* only one input file */
if (result != null)
outImage.SetField(TiffTag.PAGENUMBER, result[0], result[1]);
}
else
{
outImage.SetField(TiffTag.PAGENUMBER, m_pageNum++, 0);
}
}
else
{
if (result != null)
{
if (m_pageNum < 0)
{
/* only one input file */
outImage.SetField(TiffTag.PAGENUMBER, result[0], result[1]);
}
else
{
outImage.SetField(TiffTag.PAGENUMBER, m_pageNum++, 0);
}
}
}
int NTAGS = g_tags.Length;
for (int i = 0; i < NTAGS; i++)
{
tagToCopy p = g_tags[i];
copyTag(inImage, outImage, p.tag, p.count, p.type);
}
return pickFuncAndCopy(inImage, outImage, bitspersample, samplesperpixel, length, width);
}
private static extern IntPtr WriterProperties_Compression(IntPtr writerProperties, IntPtr path, out Compression compression);
public void send_execute_cql_query(byte[] query, Compression compression)
{
oprot_.WriteMessageBegin(new TMessage("execute_cql_query", TMessageType.Call, seqid_));
execute_cql_query_args args = new execute_cql_query_args();
args.Query = query;
args.Compression = compression;
args.Write(oprot_);
oprot_.WriteMessageEnd();
oprot_.Transport.Flush();
}
/// <summary>
/// Primary entry point to the program
/// </summary>
public static void QuantQuoteConverter(string destinationDirectory, string sourceDirectory, string resolution)
{
//Document the process:
Console.WriteLine("QuantConnect.ToolBox: QuantQuote Converter: ");
Console.WriteLine("==============================================");
Console.WriteLine("The QuantQuote converter transforms QuantQuote orders into the LEAN Algorithmic Trading Engine Data Format.");
Console.WriteLine("Parameters required: --source-dir= --destination-dir= --resolution=");
Console.WriteLine(" 1> Source Directory of Unzipped QuantQuote Data.");
Console.WriteLine(" 2> Destination Directory of LEAN Data Folder. (Typically located under Lean/Data)");
Console.WriteLine(" 3> Resolution of your QuantQuote data. (either minute, second or tick)");
Console.WriteLine(" ");
Console.WriteLine("NOTE: THIS WILL OVERWRITE ANY EXISTING FILES.");
if (sourceDirectory.IsNullOrEmpty() || destinationDirectory.IsNullOrEmpty() || resolution.IsNullOrEmpty())
{
Console.WriteLine("1. Source QuantQuote source directory: ");
sourceDirectory = (Console.ReadLine() ?? "");
Console.WriteLine("2. Destination LEAN Data directory: ");
destinationDirectory = (Console.ReadLine() ?? "");
Console.WriteLine("3. Enter Resolution (minute/second/tick): ");
resolution = (Console.ReadLine() ?? "");
resolution = resolution.ToLowerInvariant();
}
//Validate the user input:
Validate(sourceDirectory, destinationDirectory, resolution);
//Remove the final slash to make the path building easier:
sourceDirectory = StripFinalSlash(sourceDirectory);
destinationDirectory = StripFinalSlash(destinationDirectory);
//Count the total files to process:
Console.WriteLine("Counting Files...");
var count = 0;
var totalCount = GetCount(sourceDirectory);
Console.WriteLine(Invariant($"Processing {totalCount} Files ..."));
//Enumerate files
foreach (var directory in Directory.EnumerateDirectories(sourceDirectory))
{
var date = GetDate(directory);
foreach (var file in Directory.EnumerateFiles(directory))
{
var symbol = GetSymbol(file);
var fileContents = File.ReadAllText(file);
var data = new Dictionary <string, string> {
{ Invariant($"{date:yyyyMMdd}_{symbol}_Trade_Second.csv"), fileContents }
};
var fileDestination = Invariant($"{destinationDirectory}/equity/{resolution}/{symbol}/{date:yyyyMMdd}_trade.zip");
if (!Compression.ZipData(fileDestination, data))
{
Error("Error: Could not convert to Lean zip file.");
}
else
{
Console.WriteLine(Invariant($"Successfully processed {count} of {totalCount} files: {fileDestination}"));
}
count++;
}
}
Console.ReadKey();
}
internal bool setCompressionScheme(Compression scheme)
{
TiffCodec c = FindCodec(scheme);
if (c == null)
{
/*
* Don't treat an unknown compression scheme as an error.
* This permits applications to open files with data that
* the library does not have builtin support for, but which
* may still be meaningful.
*/
c = m_builtInCodecs[0];
}
m_decodestatus = c.CanDecode;
m_flags &= ~(TiffFlags.NoBitRev | TiffFlags.NoReadRaw);
m_currentCodec = c;
return c.Init();
}
public ByteFileInfo(byte[] File, string FileName)
{
this.Data = File;
this.FileID = -1;
this.FileName = FileName;
this.IsLocal = false;
this.Path = (string)null;
int num;
if (this.Data.Length > 4)
{
num = !(Convert.ToChar(this.Data[0]).ToString() + (object)Convert.ToChar(this.Data[1]) + (object)Convert.ToChar(this.Data[2]) + (object)Convert.ToChar(this.Data[3]) == "LZ77") ? 1 : 0;
}
else
{
num = 1;
}
if (num == 0)
{
this.GotLZ77Header = true;
this.IsLZ77Compressed = true;
this.Data = Compression.LZ77DecompressHeader(this.Data);
}
else if (this.Data.Length > 4 && (this.Data[0] == (byte)16 || this.Data[0] == (byte)0))
{
try
{
byte[] numArray = Compression.LZ77Decompress(this.Data);
if (numArray.Length != 0)
{
this.GotLZ77Header = false;
this.IsLZ77Compressed = true;
this.Data = numArray;
}
else
{
this.GotLZ77Header = false;
this.IsLZ77Compressed = false;
}
}
catch
{
this.GotLZ77Header = false;
this.IsLZ77Compressed = false;
}
}
if (this.Data.Length <= 4 || this.Data[0] != (byte)17 || this.IsLZ77Compressed)
{
return;
}
this.GotLZ77Header = false;
try
{
byte[] numArray = Compression.LZ11Decompress(this.Data);
if (numArray.Length != 0)
{
this.IsLZ11Compressed = true;
this.Data = numArray;
}
else
{
this.IsLZ11Compressed = false;
}
}
catch
{
this.IsLZ11Compressed = false;
}
}
private hash_t[] m_enc_hashtab; /* kept separate for small machines */
public LZWCodec(Tiff tif, Compression scheme, string name)
: base(tif, scheme, name)
{
}
public ByteFileInfo(PAZ.PAZFileEntry File)
{
this.FromFileEntry = true;
this.PAZFileEntry = File;
this.Data = File.Data;
this.FileID = -1;
this.FileName = File.Name;
this.IsLocal = false;
this.Path = (string)null;
int num;
if (this.Data.Length > 4)
{
num = !(Convert.ToChar(this.Data[0]).ToString() + (object)Convert.ToChar(this.Data[1]) + (object)Convert.ToChar(this.Data[2]) + (object)Convert.ToChar(this.Data[3]) == "LZ77") ? 1 : 0;
}
else
{
num = 1;
}
if (num == 0)
{
this.GotLZ77Header = true;
this.IsLZ77Compressed = true;
this.Data = Compression.LZ77DecompressHeader(this.Data);
}
else if (this.Data.Length > 4 && (this.Data[0] == (byte)16 || this.Data[0] == (byte)0))
{
try
{
byte[] numArray = Compression.LZ77Decompress(this.Data);
if (numArray.Length != 0)
{
this.GotLZ77Header = false;
this.IsLZ77Compressed = true;
this.Data = numArray;
}
else
{
this.GotLZ77Header = false;
this.IsLZ77Compressed = false;
}
}
catch
{
this.GotLZ77Header = false;
this.IsLZ77Compressed = false;
}
}
if (this.Data.Length > 4 && this.Data[0] == (byte)17 && !this.IsLZ77Compressed)
{
this.GotLZ77Header = false;
try
{
byte[] numArray = Compression.LZ11Decompress(this.Data);
if (numArray.Length != 0)
{
this.IsLZ11Compressed = true;
this.Data = numArray;
}
else
{
this.IsLZ11Compressed = false;
}
}
catch
{
this.IsLZ11Compressed = false;
}
}
string type = FileHandler.GetType(this);
if (!(type == "NDS") && !(type == "NARC") && !(type == "SDAT") && !(type == "PAZ"))
{
return;
}
this.IsArchive = true;
}
public PackBitsCodec(Tiff tif, Compression scheme, string name)
: base(tif, scheme, name)
{
}
/// <summary>
/// Reads ticks from a Dukascopy binary buffer into a list
/// </summary>
/// <param name="symbol">The symbol</param>
/// <param name="bytesBi5">The buffer in binary format</param>
/// <param name="date">The date for the ticks</param>
/// <param name="timeOffset">The time offset in milliseconds</param>
/// <param name="pointValue">The price multiplier</param>
private static unsafe List <Tick> AppendTicksToList(Symbol symbol, byte[] bytesBi5, DateTime date, int timeOffset, double pointValue)
{
var ticks = new List <Tick>();
byte[] bytes;
var inputFile = $"{Guid.NewGuid()}.7z";
var outputDirectory = $"{Guid.NewGuid()}";
try
{
File.WriteAllBytes(inputFile, bytesBi5);
Compression.Extract7ZipArchive(inputFile, outputDirectory);
var outputFileInfo = Directory.CreateDirectory(outputDirectory).GetFiles("*").First();
bytes = File.ReadAllBytes(outputFileInfo.FullName);
}
catch (Exception err)
{
Log.Error(err, "Failed to read raw data into stream");
return(new List <Tick>());
}
finally
{
if (File.Exists(inputFile))
{
File.Delete(inputFile);
}
if (Directory.Exists(outputDirectory))
{
Directory.Delete(outputDirectory, true);
}
}
int count = bytes.Length / DukascopyTickLength;
// Numbers are big-endian
// ii1 = milliseconds within the hour
// ii2 = AskPrice * point value
// ii3 = BidPrice * point value
// ff1 = AskVolume (not used)
// ff2 = BidVolume (not used)
fixed(byte *pBuffer = &bytes[0])
{
uint *p = (uint *)pBuffer;
for (int i = 0; i < count; i++)
{
ReverseBytes(p); uint time = *p++;
ReverseBytes(p); uint ask = *p++;
ReverseBytes(p); uint bid = *p++;
p++; p++;
if (bid > 0 && ask > 0)
{
ticks.Add(new Tick(
date.AddMilliseconds(timeOffset + time),
symbol,
Convert.ToDecimal(bid / pointValue),
Convert.ToDecimal(ask / pointValue)));
}
}
}
return(ticks);
}
public TiffDirectory()
{
td_subfiletype = 0;
td_compression = 0;
td_photometric = 0;
td_planarconfig = 0;
td_fillorder = BitOrder.BigEndian;
td_bitspersample = 1;
td_threshholding = Threshold.BILevel;
td_orientation = Orientation.TopLeft;
td_samplesperpixel = 1;
td_rowsperstrip = -1;
td_tiledepth = 1;
td_stripbytecountsorted = true; // Our own arrays always sorted.
td_resolutionunit = ResolutionUnit.Inch;
td_sampleformat = SampleFormat.UInt;
td_imagedepth = 1;
td_ycbcrsubsampling[0] = 2;
td_ycbcrsubsampling[1] = 2;
td_ycbcrpositioning = YCbCrPosition.Centered;
}
public PackBitsCodec(Tiff tif, Compression scheme, string name)
: base(tif, scheme, name)
{
}
/// <summary>
/// Checks whether library has working codec for the specific compression scheme.
/// </summary>
/// <param name="scheme">The scheme to check.</param>
/// <returns>
/// <c>true</c> if the codec is configured and working; otherwise, <c>false</c>.
/// </returns>
public bool IsCodecConfigured(Compression scheme)
{
TiffCodec codec = FindCodec(scheme);
if (codec == null)
return false;
if (codec.CanEncode != false || codec.CanDecode != false)
return true;
return false;
}
internal static void SetCompressionType(BuildTargetGroup targetGroup, Compression type)
{
SetCompressionTypeInternal(targetGroup, (int)type);
}