public virtual void Save(string fileName)
{
if (fileName == null)
{
throw new ArgumentNullException("fileName");
}
var timer = new Stopwatch();
timer.Start();
if (File.Exists(fileName))
{
using (var fs = File.Open(fileName, FileMode.Truncate, FileAccess.Write, FileShare.Read))
using (var memStream = new MemoryStream())
{
Serializer.Serialize(memStream, this);
var bytes = memStream.ToArray();
var compressed = QuickLZ.compress(bytes, 1);
fs.Write(compressed, 0, compressed.Length);
}
Log.DebugFormat("re-wrote {0} in {1}", fileName, timer.Elapsed);
}
else
{
using (var fs = File.Open(fileName, FileMode.CreateNew, FileAccess.Write, FileShare.None))
using (var memStream = new MemoryStream())
{
Serializer.Serialize(memStream, this);
var bytes = memStream.ToArray();
var compressed = QuickLZ.compress(bytes, 1);
fs.Write(compressed, 0, compressed.Length);
}
Log.DebugFormat("created {0} in {1}", fileName, timer.Elapsed);
}
}
private byte[] Serialize(T obj)
{
byte[] data;
using (var ms = new MemoryStream())
{
Serializer.Serialize(ms, obj);
data = ms.ToArray();
}
if (_isCompressed)
{
data = QuickLZ.compress(data, 3);
}
if (_isEncrypted)
{
data = AesHelper.AesEncrypt(data, GetBytes(_encKey));
}
var body = data;
var header = BitConverter.GetBytes(body.Length);
var payload = header.Concat(body).ToArray();
return(payload);
}
public static string Compress(string s)
{
var uncompressed = Encoding.Unicode.GetBytes(s);
var compressed = QuickLZ.compress(uncompressed);
return(Convert.ToBase64String(compressed));
}
public byte[] SerializeToByteArray(SerializedObject objSerializedObject)
{
byte[] bytData = null;
using (MemoryStream objTempStream = new MemoryStream())
{
BinaryFormatterKeyManager objKeyManager = new BinaryFormatterKeyManager();
BinaryWriter objBinaryRecordWriter = new BinaryWriter(objTempStream);
Serialize(objSerializedObject, objBinaryRecordWriter, objKeyManager);
objBinaryRecordWriter.Write(EndDataType);
using (MemoryStream objMemoryStream = new MemoryStream())
{
using (BinaryWriter objBinaryWriter = new BinaryWriter(objMemoryStream))
{
objKeyManager.SerializeToStream(objBinaryWriter);
objBinaryWriter.Write(objTempStream.ToArray());
}
bytData = objMemoryStream.ToArray();
}
}
byte[] bytReturnData = bytData;
if (bytData.Length >= CompressionThreshold)
{
bytReturnData = QuickLZ.compress(bytData, 1);
}
return(bytReturnData);
}
public static void Serialize(this IDictionary <short, Field> fields, Compression compression, Stream stream)
{
foreach (var field in fields)
{
byte[] keyBytes = BitConverter.GetBytes(field.Key);
byte[] valBytes;
string toStore = field.Value.Store ? field.Value.Value : string.Empty;
if (compression == Compression.GZip)
{
valBytes = Deflator.Compress(Encoding.GetBytes(toStore));
}
else if (compression == Compression.Lz)
{
valBytes = QuickLZ.compress(Encoding.GetBytes(toStore), 1);
}
else
{
valBytes = Encoding.GetBytes(toStore);
}
byte[] valLengthBytes = BitConverter.GetBytes(valBytes.Length);
if (!BitConverter.IsLittleEndian)
{
Array.Reverse(keyBytes);
Array.Reverse(valBytes);
Array.Reverse(valLengthBytes);
}
stream.Write(keyBytes, 0, sizeof(short));
stream.Write(valLengthBytes, 0, sizeof(int));
stream.Write(valBytes, 0, valBytes.Length);
}
}
public static string Serialize(Graph g)
{
var bytes = SerializeToByteArray(g);
var compressed = QuickLZ.compress(bytes);
var Ascii85 = new Ascii85();
return(Prefix + Ascii85.Encode(compressed));
}
/// <summary>
/// 加密
/// </summary>
/// <param name="sourceByteArr"></param>
/// <returns></returns>
public byte[] encryption(byte[] sourceByteArr)
{
// 压缩
byte[] compressByteArr = QuickLZ.compress(sourceByteArr, 1);
// 加密
byte[] encryptedBytes = this.encryptionByteArr(compressByteArr, tdes.Key, tdes.IV);
return(encryptedBytes);
}
private byte[] Serialize(Document doc)
{
using (var ms = new MemoryStream())
{
GraphSerializer.Serializer.Serialize(ms, doc);
var bytes = ms.ToArray();
var compressed = QuickLZ.compress(bytes, 1);
return(compressed);
}
}
public GenericFile(string filePath, bool bCompress)
{
OriginalFileData = File.ReadAllBytes(filePath);
if (bCompress)
{
OriginalFileData = QuickLZ.compress(OriginalFileData, 3);
}
setEncryptionKey();
}
public override void ProcessStep()
{
var target = Globals.Context.TargetAssembly;
Globals.Context.Options.MoveReferences = false;
foreach (var markedRef in Globals.Context.MarkedReferences)
{
var buff = QuickLZ.compress(File.ReadAllBytes(markedRef.Value), Globals.Context.Options.CompressionLevel);
var asm = markedRef.Key;
target.MainModule.Resources.Add(new EmbeddedResource(asm.Name.Name.MangleName(),
ManifestResourceAttributes.Private, buff));
Globals.Context.UIProvider.VerboseLog(string.Format("[Packing(Ref)] -> Packed reference ({0})", asm.Name.Name));
}
}
//public async Task Concat(IDictionary<long, IList<long>> offsets)
//{
// foreach (var offset in offsets)
// {
// var canonical = offset.Key;
// var batch = new Dictionary<long, IList<long>>();
// batch.Add(canonical, offset.Value);
// await ExecuteConcat(batch);
// }
//}
public async Task Concat(IDictionary <long, IList <long> > offsets)
{
if (offsets == null)
{
throw new ArgumentNullException(nameof(offsets));
}
var timer = Stopwatch.StartNew();
var requestMessage = new MemoryStream();
foreach (var offset in offsets)
{
// write key
requestMessage.Write(BitConverter.GetBytes(offset.Key));
// write count
requestMessage.Write(BitConverter.GetBytes(offset.Value.Count));
foreach (var offs in offset.Value)
{
// write data
requestMessage.Write(BitConverter.GetBytes(offs));
}
}
var messageBuf = requestMessage.ToArray();
var compressed = QuickLZ.compress(messageBuf, 3);
var endpoint = string.Format("{0}{1}?concat=true",
_config.Get("postings_endpoint"), _collectionName);
var request = (HttpWebRequest)WebRequest.Create(endpoint);
request.Method = WebRequestMethods.Http.Post;
request.ContentType = "application/postings";
using (var requestBody = await request.GetRequestStreamAsync())
{
requestBody.Write(compressed, 0, compressed.Length);
using (var response = (HttpWebResponse)await request.GetResponseAsync())
{
this.Log(string.Format("{0} concat operation took {1}", _collectionName, timer.Elapsed));
}
}
}
public byte[] Compress()
{
byte[] bytData = null;
using (MemoryStream objMemoryStream = new MemoryStream())
{
Compress(objMemoryStream);
bytData = objMemoryStream.ToArray();
}
byte[] bytCompressedData = bytData;
if (bytCompressedData.Length >= CompressionThreshold)
{
bytCompressedData = QuickLZ.compress(bytData, 1);
}
return(bytCompressedData);
}
public override void ProcessStep()
{
TypeDefinition resolver;
StubWorker.PopulateStub(ref _stubAssembly, _decompressor, _loader, out resolver);
StubWorker.StripCoreDependency(ref _stubAssembly, _decompressor, resolver);
using (var ms = new MemoryStream())
{
Globals.Context.TargetAssembly.Write(ms);
var tmpBuff = QuickLZ.compress(ms.ToArray(), Globals.Context.Options.CompressionLevel);
_stubAssembly.MainModule.Resources.Add(new EmbeddedResource("X", ManifestResourceAttributes.Public,
tmpBuff));
Globals.Context.UIProvider.VerboseLog(string.Format("[Pack(Main)] -> Packed assembly: {0}", Globals.Context.TargetAssembly.Name.Name));
}
}
public static byte[] Serialize(this IList <Field> fields, Compression compression)
{
using (var stream = new MemoryStream())
{
foreach (var field in fields)
{
byte[] keyBytes = Encoding.GetBytes(field.Key);
byte[] keyLengthBytes = BitConverter.GetBytes((short)keyBytes.Length);
byte[] valBytes;
string toStore = field.Store ? field.Value : string.Empty;
if (compression == Compression.GZip)
{
valBytes = Deflator.Compress(Encoding.GetBytes(toStore));
}
else if (compression == Compression.QuickLz)
{
valBytes = QuickLZ.compress(Encoding.GetBytes(toStore), 1);
}
else
{
valBytes = Encoding.GetBytes(toStore);
}
byte[] valLengthBytes = BitConverter.GetBytes(valBytes.Length);
if (!BitConverter.IsLittleEndian)
{
Array.Reverse(keyLengthBytes);
Array.Reverse(keyBytes);
Array.Reverse(valBytes);
Array.Reverse(valLengthBytes);
}
stream.Write(keyLengthBytes, 0, sizeof(short));
stream.Write(keyBytes, 0, keyBytes.Length);
stream.Write(valLengthBytes, 0, sizeof(int));
stream.Write(valBytes, 0, valBytes.Length);
}
return(stream.ToArray());
}
}
// Token: 0x06006B14 RID: 27412 RVA: 0x001E0AF0 File Offset: 0x001DECF0
public static ArraySegment <byte> EncodeMessage(object vMsg, IProtoProvider protoProvider)
{
byte[] array = new byte[65536];
MemoryStream memoryStream = new MemoryStream(array);
BinaryWriter binaryWriter = new BinaryWriter(memoryStream);
binaryWriter.Write(0);
binaryWriter.Write(0);
RuntimeTypeModel.Default.Serialize(memoryStream, vMsg, null);
bool flag = memoryStream.Position - 4L >= 400L;
ushort num;
if (flag)
{
using (MemoryStream memoryStream2 = new MemoryStream(memoryStream.ToArray(), 4, (int)memoryStream.Position - 4))
{
byte[] array2 = QuickLZ.compress(memoryStream2.ToArray(), 3);
memoryStream.Seek(4L, SeekOrigin.Begin);
memoryStream.Write(array2, 0, array2.Length);
}
num = (ushort)memoryStream.Position;
binaryWriter.Seek(0, SeekOrigin.Begin);
binaryWriter.Write(num);
ushort num2 = (ushort)protoProvider.GetIdByType(vMsg.GetType());
num2 |= 32768;
binaryWriter.Write(num2);
}
else
{
num = (ushort)memoryStream.Position;
binaryWriter.Seek(0, SeekOrigin.Begin);
binaryWriter.Write(num);
ushort num3 = (ushort)protoProvider.GetIdByType(vMsg.GetType());
num3 &= 32767;
binaryWriter.Write(num3);
}
return(new ArraySegment <byte>(array, 0, (int)num));
}
public async Task Write(VectorNode rootNode)
{
var timer = Stopwatch.StartNew();
IList <VectorNode> nodes;
byte[] payload;
// create postings message
using (var message = new MemoryStream())
using (var lengths = new MemoryStream())
using (var offsets = new MemoryStream())
using (var documents = new MemoryStream())
{
// Write length of word (i.e. length of list of postings) to header stream,
// postings offsets to offset stream,
// and word itself to documents stream.
nodes = rootNode.SerializePostings(lengths, offsets, documents);
if (nodes.Count == 0)
{
return;
}
if (nodes.Count != lengths.Length / sizeof(int))
{
throw new DataMisalignedException();
}
// first word of message is payload count (i.e. num of postings lists)
await message.WriteAsync(BitConverter.GetBytes(nodes.Count));
// next are lengths
lengths.Position = 0;
await lengths.CopyToAsync(message);
// then all of the offsets
offsets.Position = 0;
await offsets.CopyToAsync(message);
// last are the document IDs
documents.Position = 0;
await documents.CopyToAsync(message);
var buf = message.ToArray();
var ctime = Stopwatch.StartNew();
var compressed = QuickLZ.compress(buf, 3);
this.Log(string.Format("compressing {0} bytes to {1} took {2}", buf.Length, compressed.Length, ctime.Elapsed));
payload = compressed;
}
this.Log(string.Format("create postings message took {0}", timer.Elapsed));
// send message, recieve list of (remote) file positions, save positions in index.
var positions = await Send(payload);
if (nodes.Count != positions.Count)
{
throw new DataMisalignedException();
}
timer.Restart();
for (int i = 0; i < nodes.Count; i++)
{
nodes[i].PostingsOffset = positions[i];
}
this.Log(string.Format("record postings offsets took {0}", timer.Elapsed));
}
public byte[] Compress(byte[] source)
{
return(QuickLZ.compress(source, 1));
}