public object DeserializeFromJsonStream(string json)
{
return(SerializerHelper <T> .DeserializeFromJsonStream(json));
}
public object DeserializeFromByteStream(MemoryStream memStream)
{
return(SerializerHelper <T> .DeserializeFromByteStream(memStream));
}
protected long UpdateHelper(IKey key, IValue value, bool IsAppend, Byte[] valHash = null, long timestamp = -1, long offsetInStream = -1)
{
long offset;
if (!key.GetType().Equals(typeof(KeyType)))
{
throw new InvalidDataException("Invalid IKey Type");
}
if (!value.GetType().Equals(typeof(ValType)))
{
throw new InvalidDataException("Invalid IValue Type");
}
if (logger != null)
{
logger.Log("Start ValueDataStream Tag Lookup");
}
List <DataBlockInfo> offsets;
// check if the entry is present (and record offset)
if (index.ContainsKey(key))
{
// TODO: this could just have been a simple collision
// -- make sure that this is the same key
// -- if different, use a secondary hash function
offsets = index[key];
}
else
{
offsets = new List <DataBlockInfo>();
index[key] = offsets;
// IncrementIndexSize(offsets.Count);
IncrementIndexSize(1);
}
if (logger != null)
{
logger.Log("End ValueDataStream Tag Lookup");
}
if (logger != null)
{
logger.Log("Start ValueDataStream Construct DataBlock");
}
// write <op (1B), ts, key, val_len, val>
byte op;
if (IsAppend)
{
op = (byte)WriteOp.AppendOp;
}
else
{
op = (byte)WriteOp.UpdateOp;
}
long ts;
// Construct datablock
if (timestamp == -1)
{
ts = StreamFactory.NowUtc();
}
else
{
ts = timestamp;
}
DataBlock <KeyType, ValType> db = new DataBlock <KeyType, ValType>();
db.op = op;
//db.timestamp = ts;
//db.setKey(key);
/*
* Commenting out because the synchronizer will do this now at the chunk level
* if (streamtype == StreamFactory.StreamSecurityType.Secure)
* {
* //if (logger != null) logger.Log("Start encrypting value");
* db.value = new ByteValue(Crypto.EncryptBytesSimple(value.GetBytes(), Crypto.KeyDer(acl_md.encKey), acl_md.IV));
* //if (logger != null) logger.Log("End encrypting value");
* }
* else*/
{
//db.value = new ByteValue(value.GetBytes());
db.setValue(value);
}
if (logger != null)
{
logger.Log("End ValueDataStream Construct DataBlock");
}
if (offsetInStream == -1)
{
if (logger != null)
{
logger.Log("Start ValueDataStream Serialize DataBlock");
}
//Byte[] buffer = db.SerializeToByteStream().ToArray();
Byte[] buffer = SerializerHelper <DataBlock <KeyType, ValType> > .SerializeToProtoStream(db).ToArray();
if (logger != null)
{
logger.Log("End ValueDataStream Serialize DataBlock");
}
if (logger != null)
{
logger.Log("Start ValueDataStream WriteToDisc DataBlock");
}
// fs_bw.Write(db.SerializeToJsonStream());
// get file offset; add <key_hash, offset> to index
fs_bw.BaseStream.Seek(0, SeekOrigin.End);
offset = fs_bw.BaseStream.Position;
fs_bw.Write(buffer.Length);
fs_bw.Write(buffer);
fs_bw.Flush();
if (logger != null)
{
logger.Log("End ValueDataStream WriteToDisc DataBlock");
}
}
else
{
offset = offsetInStream;
}
// Construct dbinfo in index
if (logger != null)
{
logger.Log("Start ValueDataStream Construct and Add DataBlockInfo");
}
DataBlockInfo latest_tso = new DataBlockInfo(ts, offset);
if (streamtype == StreamFactory.StreamSecurityType.Secure)
{
if (valHash != null) // passed from the dir stream and written into datablockinfo. not written for file stream
{
latest_tso.hashValue = valHash;
}
/*
* else
* {
* if (logger != null) logger.Log("Start taking hash of encrypted value");
* latest_tso.hashValue = hasher.ComputeHash(db.value.GetBytes());
* if (logger != null) logger.Log("End taking hash of encrypted value");
*
* }
* latest_tso.key_version = acl_md.keyVersion;
*/
}
// apply to index
if (IsAppend)
{
offsets.Add(latest_tso);
// IncrementIndexSize();
}
else
{
if (offsets.Count == 0)
{
offsets.Add(latest_tso);
// IncrementIndexSize();
}
else
{
offsets[offsets.Count - 1] = latest_tso;
}
}
if (logger != null)
{
logger.Log("End ValueDataStream Construct and Add DataBlockInfo");
}
return(offset);
}
internal DataBlock <KeyType, ValType> ReadDataBlock(long offset, bool ignoreIntegrity = false)
{
if (logger != null)
{
logger.Log("Start ValueDataStream GetDB");
}
int data_len;
byte[] data;
try
{
if (logger != null)
{
logger.Log("Start ValueDataStream ReadData");
}
if (fs_br == null) // remote read
{
synchronizer.SetDataFileName(DataLogFileName);
int dataLength = BitConverter.ToInt32(synchronizer.ReadData(offset, 4), 0);
data = synchronizer.ReadData(offset + 4, dataLength);
}
else
{
if (logger != null)
{
logger.Log("Start ValueDataStream ReadFromDisk");
}
fs_br.BaseStream.Seek(offset, SeekOrigin.Begin);
data_len = fs_br.ReadInt32();
data = fs_br.ReadBytes(data_len);
if (logger != null)
{
logger.Log("End ValueDataStream ReadFromDisk");
}
}
if (logger != null)
{
logger.Log("End ValueDataStream ReadData");
}
if (logger != null)
{
logger.Log("Start ValueDataStream Deserialize DataBlock");
}
//DataBlock<KeyType, ValType> db = SerializerHelper<DataBlock<KeyType, ValType>>.DeserializeFromByteStream(new MemoryStream(data));
DataBlock <KeyType, ValType> db = SerializerHelper <DataBlock <KeyType, ValType> > .DeserializeFromProtoStream(new MemoryStream(data));
if (logger != null)
{
logger.Log("End ValueDataStream Deserialize DataBlock");
}
/*if (streamtype == StreamFactory.StreamSecurityType.Secure && ignoreIntegrity == false)
* {
* if (logger != null) logger.Log("Start verifying hash of db within segment");
* // Commenting out because decryption is carried out by the syncronizer
* // if (!hasher.ComputeHash(db.value.GetBytes()).SequenceEqual(dbi.hashValue))
* // return null;
* if (logger != null) logger.Log("End verifying hash of db within segment");
* }
*
* if (streamtype == StreamFactory.StreamSecurityType.Secure)
* {
* if (logger != null) logger.Log("Start decrypting db within index");
* // Commenting out because decryption is carried out by the syncronizer
* //db.value = new ByteValue(Crypto.DecryptBytesSimple(db.value.GetBytes(), Crypto.KeyDer(Crypto.GetSpecificKey(acl_md.encKey, acl_md.keyVersion, dbi.key_version)), acl_md.IV));
* if (logger != null) logger.Log("End decrypting db within index");
* }*/
if (logger != null)
{
logger.Log("End ValueDataStream GetDB");
}
return(db);
}
catch (Exception e)
{
Console.WriteLine("{0} Exception caught.", e);
if (logger != null)
{
logger.Log("End ValueDataStream GetDB");
}
return(null);
}
}
protected bool FillIndex()
{
try
{
string IndexFQN = targetDir + "/" + IndexFileName;
//Console.WriteLine("***************** filling index : "+ IndexFQN);
if (File.Exists(IndexFQN))
{
FileStream iout = new FileStream(IndexFQN, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
BinaryReader index_br = new BinaryReader(iout);
long latest_ts = 0;
try
{
while (true)
{
string key = index_br.ReadString();
IKey ikey = SerializerHelper <KeyType> .DeserializeFromJsonStream(key) as IKey;
int num_offsets = index_br.ReadInt32();
List <DataBlockInfo> offsets = new List <DataBlockInfo>(num_offsets);
for (int i = 0; i < num_offsets; i++)
{
long ts = index_br.ReadInt64();
long offset = index_br.ReadInt64();
DataBlockInfo tso = new DataBlockInfo(ts, offset);
if (streamtype == StreamFactory.StreamSecurityType.Secure)
{
Int32 h_len = index_br.ReadInt32();
if (h_len > 0)
{
tso.hashValue = index_br.ReadBytes(h_len);
}
}
offsets.Add(tso);
if (ts > latest_ts)
{
latest_ts = ts;
latest_key = ikey;
}
}
index[ikey] = offsets;
IncrementIndexSize(offsets.Count);
}
}
catch (Exception)
{
// done
// }
// finally
// {
if (streamtype == StreamFactory.StreamSecurityType.Secure)
{
iout.Position = 0;
IndexHash = hasher.ComputeHash(iout);
}
// Console.WriteLine("********************* closed " + IndexFQN);
index_br.Close();
iout.Close();
GC.Collect();
}
}
else
{
return(false);
}
}
catch (Exception exp)
{
Console.WriteLine("{0} Exception caught.", exp);
return(false);
}
return(true);
}
//***
private void BuildIndexHeader()
{
FileStream iout;
BinaryReader index_br;
indexHeader = new Dictionary <IKey, long>();
string IndexFilePath = targetDir + "/" + IndexFileName;
if (!File.Exists(IndexFilePath))
{
return;
}
//Console.WriteLine(DateTime.Now+" ******** filling indexheader " + IndexFilePath);
iout = new FileStream(IndexFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
index_br = new BinaryReader(iout);
try
{
while (true)
{
long keyOffset = index_br.BaseStream.Position;
string key = index_br.ReadString();
IKey ikey = SerializerHelper <KeyType> .DeserializeFromJsonStream(key) as IKey;
indexHeader[ikey] = keyOffset;
int num_offsets = index_br.ReadInt32();
for (int i = 0; i < num_offsets; i++)
{
index_br.ReadBytes(16);//timestamp and offset
if (streamtype == StreamFactory.StreamSecurityType.Secure)
{
index_br.ReadUInt32(); //key_version
Int32 h_len = index_br.ReadInt32(); //h_len?
index_br.ReadBytes(h_len); //hashvalue
}
}
}
}
catch (Exception)
{
// done
// }
// catch(Exception e)
// {
// Console.Write("Exception in buildindexheader: "+e);
// }
// finally
// {
if (streamtype == StreamFactory.StreamSecurityType.Secure)
{
iout.Position = 0;
IndexHash = hasher.ComputeHash(iout);
}
GC.Collect();
index_br.Close();
iout.Close();
//Console.WriteLine(DateTime.Now+" ******** closed indexheader " + IndexFilePath);
GC.Collect();
}
}
