/// <summary> /// </summary> /// <param name="fname"> /// </param> /// <typeparam name="T"> /// </typeparam> /// <returns> /// </returns> public static List <T> UncompressData <T>(string fname) { // Need to build the serializer/deserializer prior to Task invocations MessagePackSerializer <List <T> > constructor = MessagePackSerializer.Create <List <T> >(); List <T> resultList = new List <T>(); using (Stream fileStream = new FileStream(fname, FileMode.Open)) { BinaryReader binaryReader = new BinaryReader(fileStream); byte versionlength = binaryReader.ReadByte(); char[] version = new char[versionlength]; version = binaryReader.ReadChars(versionlength); string versionString = ""; foreach (char c in version) { versionString += c; } Console.WriteLine("Loading data for client version " + versionString); // TODO: Check version and print a warning if not same as config.xml's // packaged is unused here int packaged = binaryReader.ReadInt32(); int capacity = binaryReader.ReadInt32(); int slices = binaryReader.ReadInt32(); TaskedSerializer <T>[] tasked = new TaskedSerializer <T> [slices]; Task[] tasks = new Task[slices]; for (int i = 0; i < slices; i++) { int size = binaryReader.ReadInt32(); byte[] tempBuffer = binaryReader.ReadBytes(size); using (MemoryStream tempStream = new MemoryStream(tempBuffer)) { tasked[i] = new TaskedSerializer <T>(tempStream); } int i1 = i; tasks[i] = new Task(() => tasked[i1].Deserialize()); tasks[i].Start(); } Task.WaitAll(tasks); resultList = new List <T>(capacity); for (int i = 0; i < slices; i++) { resultList.AddRange(tasked[i].DataSlice); tasked[i].DataSlice.Clear(); tasks[i].Dispose(); tasked[i].Dispose(); } } return(resultList); }
/// <summary> /// </summary> /// <param name="filename"> /// </param> /// <param name="version"> /// </param> /// <param name="dataList"> /// </param> /// <param name="packCount"> /// </param> /// <typeparam name="T"> /// </typeparam> /// <exception cref="Exception"> /// </exception> public static void CompressData <T>(string filename, string version, List <T> dataList, int packCount = 500) { // Need to build the serializer/deserializer prior to Task invocations MessagePackSerializer <List <T> > constructor = MessagePackSerializer.Create <List <T> >(); Console.WriteLine("Compressing " + typeof(T).Name + "s"); if (packCount == 0) { throw new Exception("Dont use 0 as packCount!!"); } using (Stream fileStream = new FileStream(filename, FileMode.Create)) { BinaryWriter binaryWriter = new BinaryWriter(fileStream); byte[] versionbuffer = Encoding.ASCII.GetBytes(version); binaryWriter.Write((byte)versionbuffer.Length); binaryWriter.Write(versionbuffer, 0, versionbuffer.Length); binaryWriter.Write(packCount); int tempCapacity = dataList.Count; binaryWriter.Write(tempCapacity); int maxCount = dataList.Count; // Write number of slices int slices = Convert.ToInt32(Math.Ceiling((double)maxCount / packCount)); binaryWriter.Write(slices); TaskedSerializer <T>[] taskData = new TaskedSerializer <T> [slices]; Task[] tasks = new Task[taskData.Length]; for (int i = 0; i < taskData.Count(); i++) { taskData[i] = new TaskedSerializer <T>(dataList.Skip(packCount * i).Take(packCount).ToList()); int i1 = i; tasks[i] = new Task(() => taskData[i1].Serialize()); tasks[i].Start(); } // Wait for all serialization to finish Task.WaitAll(tasks); Console.WriteLine("100% serialized"); // Write data streams foreach (TaskedSerializer <T> task in taskData) { task.Stream.Position = 0; int size = (int)task.Stream.Length; binaryWriter.Write(size); task.Stream.CopyTo(fileStream); task.Dispose(); } for (int i = 0; i < slices; i++) { taskData[i] = null; tasks[i].Dispose(); } } }