public object DeserializeFromJsonStream(string json) { return(SerializerHelper <T> .DeserializeFromJsonStream(json)); }
public object DeserializeFromByteStream(MemoryStream memStream) { return(SerializerHelper <T> .DeserializeFromByteStream(memStream)); }
protected long UpdateHelper(IKey key, IValue value, bool IsAppend, Byte[] valHash = null, long timestamp = -1, long offsetInStream = -1) { long offset; if (!key.GetType().Equals(typeof(KeyType))) { throw new InvalidDataException("Invalid IKey Type"); } if (!value.GetType().Equals(typeof(ValType))) { throw new InvalidDataException("Invalid IValue Type"); } if (logger != null) { logger.Log("Start ValueDataStream Tag Lookup"); } List <DataBlockInfo> offsets; // check if the entry is present (and record offset) if (index.ContainsKey(key)) { // TODO: this could just have been a simple collision // -- make sure that this is the same key // -- if different, use a secondary hash function offsets = index[key]; } else { offsets = new List <DataBlockInfo>(); index[key] = offsets; // IncrementIndexSize(offsets.Count); IncrementIndexSize(1); } if (logger != null) { logger.Log("End ValueDataStream Tag Lookup"); } if (logger != null) { logger.Log("Start ValueDataStream Construct DataBlock"); } // write <op (1B), ts, key, val_len, val> byte op; if (IsAppend) { op = (byte)WriteOp.AppendOp; } else { op = (byte)WriteOp.UpdateOp; } long ts; // Construct datablock if (timestamp == -1) { ts = StreamFactory.NowUtc(); } else { ts = timestamp; } DataBlock <KeyType, ValType> db = new DataBlock <KeyType, ValType>(); db.op = op; //db.timestamp = ts; //db.setKey(key); /* * Commenting out because the synchronizer will do this now at the chunk level * if (streamtype == StreamFactory.StreamSecurityType.Secure) * { * //if (logger != null) logger.Log("Start encrypting value"); * db.value = new ByteValue(Crypto.EncryptBytesSimple(value.GetBytes(), Crypto.KeyDer(acl_md.encKey), acl_md.IV)); * //if (logger != null) logger.Log("End encrypting value"); * } * else*/ { //db.value = new ByteValue(value.GetBytes()); db.setValue(value); } if (logger != null) { logger.Log("End ValueDataStream Construct DataBlock"); } if (offsetInStream == -1) { if (logger != null) { logger.Log("Start ValueDataStream Serialize DataBlock"); } //Byte[] buffer = db.SerializeToByteStream().ToArray(); Byte[] buffer = SerializerHelper <DataBlock <KeyType, ValType> > .SerializeToProtoStream(db).ToArray(); if (logger != null) { logger.Log("End ValueDataStream Serialize DataBlock"); } if (logger != null) { logger.Log("Start ValueDataStream WriteToDisc DataBlock"); } // fs_bw.Write(db.SerializeToJsonStream()); // get file offset; add <key_hash, offset> to index fs_bw.BaseStream.Seek(0, SeekOrigin.End); offset = fs_bw.BaseStream.Position; fs_bw.Write(buffer.Length); fs_bw.Write(buffer); fs_bw.Flush(); if (logger != null) { logger.Log("End ValueDataStream WriteToDisc DataBlock"); } } else { offset = offsetInStream; } // Construct dbinfo in index if (logger != null) { logger.Log("Start ValueDataStream Construct and Add DataBlockInfo"); } DataBlockInfo latest_tso = new DataBlockInfo(ts, offset); if (streamtype == StreamFactory.StreamSecurityType.Secure) { if (valHash != null) // passed from the dir stream and written into datablockinfo. not written for file stream { latest_tso.hashValue = valHash; } /* * else * { * if (logger != null) logger.Log("Start taking hash of encrypted value"); * latest_tso.hashValue = hasher.ComputeHash(db.value.GetBytes()); * if (logger != null) logger.Log("End taking hash of encrypted value"); * * } * latest_tso.key_version = acl_md.keyVersion; */ } // apply to index if (IsAppend) { offsets.Add(latest_tso); // IncrementIndexSize(); } else { if (offsets.Count == 0) { offsets.Add(latest_tso); // IncrementIndexSize(); } else { offsets[offsets.Count - 1] = latest_tso; } } if (logger != null) { logger.Log("End ValueDataStream Construct and Add DataBlockInfo"); } return(offset); }
internal DataBlock <KeyType, ValType> ReadDataBlock(long offset, bool ignoreIntegrity = false) { if (logger != null) { logger.Log("Start ValueDataStream GetDB"); } int data_len; byte[] data; try { if (logger != null) { logger.Log("Start ValueDataStream ReadData"); } if (fs_br == null) // remote read { synchronizer.SetDataFileName(DataLogFileName); int dataLength = BitConverter.ToInt32(synchronizer.ReadData(offset, 4), 0); data = synchronizer.ReadData(offset + 4, dataLength); } else { if (logger != null) { logger.Log("Start ValueDataStream ReadFromDisk"); } fs_br.BaseStream.Seek(offset, SeekOrigin.Begin); data_len = fs_br.ReadInt32(); data = fs_br.ReadBytes(data_len); if (logger != null) { logger.Log("End ValueDataStream ReadFromDisk"); } } if (logger != null) { logger.Log("End ValueDataStream ReadData"); } if (logger != null) { logger.Log("Start ValueDataStream Deserialize DataBlock"); } //DataBlock<KeyType, ValType> db = SerializerHelper<DataBlock<KeyType, ValType>>.DeserializeFromByteStream(new MemoryStream(data)); DataBlock <KeyType, ValType> db = SerializerHelper <DataBlock <KeyType, ValType> > .DeserializeFromProtoStream(new MemoryStream(data)); if (logger != null) { logger.Log("End ValueDataStream Deserialize DataBlock"); } /*if (streamtype == StreamFactory.StreamSecurityType.Secure && ignoreIntegrity == false) * { * if (logger != null) logger.Log("Start verifying hash of db within segment"); * // Commenting out because decryption is carried out by the syncronizer * // if (!hasher.ComputeHash(db.value.GetBytes()).SequenceEqual(dbi.hashValue)) * // return null; * if (logger != null) logger.Log("End verifying hash of db within segment"); * } * * if (streamtype == StreamFactory.StreamSecurityType.Secure) * { * if (logger != null) logger.Log("Start decrypting db within index"); * // Commenting out because decryption is carried out by the syncronizer * //db.value = new ByteValue(Crypto.DecryptBytesSimple(db.value.GetBytes(), Crypto.KeyDer(Crypto.GetSpecificKey(acl_md.encKey, acl_md.keyVersion, dbi.key_version)), acl_md.IV)); * if (logger != null) logger.Log("End decrypting db within index"); * }*/ if (logger != null) { logger.Log("End ValueDataStream GetDB"); } return(db); } catch (Exception e) { Console.WriteLine("{0} Exception caught.", e); if (logger != null) { logger.Log("End ValueDataStream GetDB"); } return(null); } }
protected bool FillIndex() { try { string IndexFQN = targetDir + "/" + IndexFileName; //Console.WriteLine("***************** filling index : "+ IndexFQN); if (File.Exists(IndexFQN)) { FileStream iout = new FileStream(IndexFQN, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); BinaryReader index_br = new BinaryReader(iout); long latest_ts = 0; try { while (true) { string key = index_br.ReadString(); IKey ikey = SerializerHelper <KeyType> .DeserializeFromJsonStream(key) as IKey; int num_offsets = index_br.ReadInt32(); List <DataBlockInfo> offsets = new List <DataBlockInfo>(num_offsets); for (int i = 0; i < num_offsets; i++) { long ts = index_br.ReadInt64(); long offset = index_br.ReadInt64(); DataBlockInfo tso = new DataBlockInfo(ts, offset); if (streamtype == StreamFactory.StreamSecurityType.Secure) { Int32 h_len = index_br.ReadInt32(); if (h_len > 0) { tso.hashValue = index_br.ReadBytes(h_len); } } offsets.Add(tso); if (ts > latest_ts) { latest_ts = ts; latest_key = ikey; } } index[ikey] = offsets; IncrementIndexSize(offsets.Count); } } catch (Exception) { // done // } // finally // { if (streamtype == StreamFactory.StreamSecurityType.Secure) { iout.Position = 0; IndexHash = hasher.ComputeHash(iout); } // Console.WriteLine("********************* closed " + IndexFQN); index_br.Close(); iout.Close(); GC.Collect(); } } else { return(false); } } catch (Exception exp) { Console.WriteLine("{0} Exception caught.", exp); return(false); } return(true); }
//*** private void BuildIndexHeader() { FileStream iout; BinaryReader index_br; indexHeader = new Dictionary <IKey, long>(); string IndexFilePath = targetDir + "/" + IndexFileName; if (!File.Exists(IndexFilePath)) { return; } //Console.WriteLine(DateTime.Now+" ******** filling indexheader " + IndexFilePath); iout = new FileStream(IndexFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); index_br = new BinaryReader(iout); try { while (true) { long keyOffset = index_br.BaseStream.Position; string key = index_br.ReadString(); IKey ikey = SerializerHelper <KeyType> .DeserializeFromJsonStream(key) as IKey; indexHeader[ikey] = keyOffset; int num_offsets = index_br.ReadInt32(); for (int i = 0; i < num_offsets; i++) { index_br.ReadBytes(16);//timestamp and offset if (streamtype == StreamFactory.StreamSecurityType.Secure) { index_br.ReadUInt32(); //key_version Int32 h_len = index_br.ReadInt32(); //h_len? index_br.ReadBytes(h_len); //hashvalue } } } } catch (Exception) { // done // } // catch(Exception e) // { // Console.Write("Exception in buildindexheader: "+e); // } // finally // { if (streamtype == StreamFactory.StreamSecurityType.Secure) { iout.Position = 0; IndexHash = hasher.ComputeHash(iout); } GC.Collect(); index_br.Close(); iout.Close(); //Console.WriteLine(DateTime.Now+" ******** closed indexheader " + IndexFilePath); GC.Collect(); } }