public new IValue Get(IKey key)
{
if (logger != null)
{
logger.Log("Start FileDataStream Retreive FilePathValue");
}
IValue valuePath = base.Get(key, true);
DataBlockInfo dbi = base.GetDBI(key);
if (logger != null)
{
logger.Log("End FileDataStream Retreive FilePathValue");
}
return(ReadData(valuePath, dbi));
}
public Tuple <IValue, long> GetLatest(IKey tag)
{
Tuple <IValue, long> latest_tuple = null;
if (tag != null)
{
IValue value = Get(tag);
if (value != null)
{
DataBlockInfo dbi = GetDBI(tag);
latest_tuple = new Tuple <IValue, long>(value, dbi.ts);
}
}
return(latest_tuple);
}
private List <DataBlockInfo> GetOffsetListFromHeader(IKey key)
{
if (!indexHeader.ContainsKey(key))
{
return(null);
}
long keyOffset = indexHeader[key];
string IndexFilePath = targetDir + "/" + IndexFileName;
if (!File.Exists(IndexFilePath))
{
throw new Exception("index file is missing!!!!! ");
}
FileStream iout = new FileStream(IndexFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
BinaryReader index_br = new BinaryReader(iout);
index_br.BaseStream.Seek(keyOffset, SeekOrigin.Begin);
string keyInFile = index_br.ReadString();// increment file pointer
//IKey ikey = SerializerHelper<KeyType>.DeserializeFromJsonStream(keyInFile) as IKey;
// TODO compare with Ikey above
int num_offsets = index_br.ReadInt32();
List <DataBlockInfo> offsets = new List <DataBlockInfo>(num_offsets);
for (int i = 0; i < num_offsets; i++)
{
long ts = index_br.ReadInt64();
long offset = index_br.ReadInt64();
DataBlockInfo tso = new DataBlockInfo(ts, offset);
if (streamtype == StreamFactory.StreamSecurityType.Secure)
{
Int32 h_len = index_br.ReadInt32();
if (h_len > 0)
{
tso.hashValue = index_br.ReadBytes(h_len);
}
}
offsets.Add(tso);
}
return(offsets);
}
/// <summary>
/// ValuePath is just the value file name ts.dat and not fully qualified file path
/// </summary>
/// <param name="valuePath"></param>
/// <param name="dbi"></param>
/// <returns></returns>
internal ByteValue ReadData(IValue valuePath, DataBlockInfo dbi)
{
ByteValue byteValue = null;
string FQValuePath = targetDir + "/" + valuePath;
if (logger != null)
{
logger.Log("Start Synchronizer Simple Download");
}
if (null != valuePath && remoteRead)
{
if (!synchronizer.DownloadFile(valuePath.ToString(), FQValuePath))
{
return(byteValue);
}
}
if (logger != null)
{
logger.Log("End Synchronizer Simple Download");
}
if (logger != null)
{
logger.Log("Start FileDataStream ReadFromDisk");
}
if (null != valuePath)
{
FileStream fout = new FileStream(FQValuePath,
FileMode.OpenOrCreate,
FileAccess.Read,
FileShare.ReadWrite);
fout.Seek(0, SeekOrigin.Begin);
byte[] bytes = new byte[fout.Length];
//read file to MemoryStream
int bytesRead = 0;
fout.Read(bytes, bytesRead, (int)fout.Length);
fout.Close();
byteValue = new ByteValue(bytes);
}
if (logger != null)
{
logger.Log("End FileDataStream ReadFromDisk");
}
if (streamtype == StreamFactory.StreamSecurityType.Secure)
{
if (logger != null)
{
logger.Log("Start FileDataStream Decrypt DataBlock");
}
if (!hasher.ComputeHash(byteValue.GetBytes()).SequenceEqual(dbi.hashValue))
{
return(null);
}
byteValue = new ByteValue(Crypto.DecryptBytesSimple(byteValue.GetBytes(), Crypto.KeyDer(acl_md.encKey), acl_md.IV));
if (logger != null)
{
logger.Log("End FileDataStream Decrypt DataBlock");
}
}
return(byteValue);
}
protected long UpdateHelper(IKey key, IValue value, bool IsAppend, Byte[] valHash = null, long timestamp = -1, long offsetInStream = -1)
{
long offset;
if (!key.GetType().Equals(typeof(KeyType)))
{
throw new InvalidDataException("Invalid IKey Type");
}
if (!value.GetType().Equals(typeof(ValType)))
{
throw new InvalidDataException("Invalid IValue Type");
}
if (logger != null)
{
logger.Log("Start ValueDataStream Tag Lookup");
}
List <DataBlockInfo> offsets;
// check if the entry is present (and record offset)
if (index.ContainsKey(key))
{
// TODO: this could just have been a simple collision
// -- make sure that this is the same key
// -- if different, use a secondary hash function
offsets = index[key];
}
else
{
offsets = new List <DataBlockInfo>();
index[key] = offsets;
// IncrementIndexSize(offsets.Count);
IncrementIndexSize(1);
}
if (logger != null)
{
logger.Log("End ValueDataStream Tag Lookup");
}
if (logger != null)
{
logger.Log("Start ValueDataStream Construct DataBlock");
}
// write <op (1B), ts, key, val_len, val>
byte op;
if (IsAppend)
{
op = (byte)WriteOp.AppendOp;
}
else
{
op = (byte)WriteOp.UpdateOp;
}
long ts;
// Construct datablock
if (timestamp == -1)
{
ts = StreamFactory.NowUtc();
}
else
{
ts = timestamp;
}
DataBlock <KeyType, ValType> db = new DataBlock <KeyType, ValType>();
db.op = op;
//db.timestamp = ts;
//db.setKey(key);
/*
* Commenting out because the synchronizer will do this now at the chunk level
* if (streamtype == StreamFactory.StreamSecurityType.Secure)
* {
* //if (logger != null) logger.Log("Start encrypting value");
* db.value = new ByteValue(Crypto.EncryptBytesSimple(value.GetBytes(), Crypto.KeyDer(acl_md.encKey), acl_md.IV));
* //if (logger != null) logger.Log("End encrypting value");
* }
* else*/
{
//db.value = new ByteValue(value.GetBytes());
db.setValue(value);
}
if (logger != null)
{
logger.Log("End ValueDataStream Construct DataBlock");
}
if (offsetInStream == -1)
{
if (logger != null)
{
logger.Log("Start ValueDataStream Serialize DataBlock");
}
//Byte[] buffer = db.SerializeToByteStream().ToArray();
Byte[] buffer = SerializerHelper <DataBlock <KeyType, ValType> > .SerializeToProtoStream(db).ToArray();
if (logger != null)
{
logger.Log("End ValueDataStream Serialize DataBlock");
}
if (logger != null)
{
logger.Log("Start ValueDataStream WriteToDisc DataBlock");
}
// fs_bw.Write(db.SerializeToJsonStream());
// get file offset; add <key_hash, offset> to index
fs_bw.BaseStream.Seek(0, SeekOrigin.End);
offset = fs_bw.BaseStream.Position;
fs_bw.Write(buffer.Length);
fs_bw.Write(buffer);
fs_bw.Flush();
if (logger != null)
{
logger.Log("End ValueDataStream WriteToDisc DataBlock");
}
}
else
{
offset = offsetInStream;
}
// Construct dbinfo in index
if (logger != null)
{
logger.Log("Start ValueDataStream Construct and Add DataBlockInfo");
}
DataBlockInfo latest_tso = new DataBlockInfo(ts, offset);
if (streamtype == StreamFactory.StreamSecurityType.Secure)
{
if (valHash != null) // passed from the dir stream and written into datablockinfo. not written for file stream
{
latest_tso.hashValue = valHash;
}
/*
* else
* {
* if (logger != null) logger.Log("Start taking hash of encrypted value");
* latest_tso.hashValue = hasher.ComputeHash(db.value.GetBytes());
* if (logger != null) logger.Log("End taking hash of encrypted value");
*
* }
* latest_tso.key_version = acl_md.keyVersion;
*/
}
// apply to index
if (IsAppend)
{
offsets.Add(latest_tso);
// IncrementIndexSize();
}
else
{
if (offsets.Count == 0)
{
offsets.Add(latest_tso);
// IncrementIndexSize();
}
else
{
offsets[offsets.Count - 1] = latest_tso;
}
}
if (logger != null)
{
logger.Log("End ValueDataStream Construct and Add DataBlockInfo");
}
return(offset);
}
protected bool FillIndex()
{
try
{
string IndexFQN = targetDir + "/" + IndexFileName;
//Console.WriteLine("***************** filling index : "+ IndexFQN);
if (File.Exists(IndexFQN))
{
FileStream iout = new FileStream(IndexFQN, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
BinaryReader index_br = new BinaryReader(iout);
long latest_ts = 0;
try
{
while (true)
{
string key = index_br.ReadString();
IKey ikey = SerializerHelper <KeyType> .DeserializeFromJsonStream(key) as IKey;
int num_offsets = index_br.ReadInt32();
List <DataBlockInfo> offsets = new List <DataBlockInfo>(num_offsets);
for (int i = 0; i < num_offsets; i++)
{
long ts = index_br.ReadInt64();
long offset = index_br.ReadInt64();
DataBlockInfo tso = new DataBlockInfo(ts, offset);
if (streamtype == StreamFactory.StreamSecurityType.Secure)
{
Int32 h_len = index_br.ReadInt32();
if (h_len > 0)
{
tso.hashValue = index_br.ReadBytes(h_len);
}
}
offsets.Add(tso);
if (ts > latest_ts)
{
latest_ts = ts;
latest_key = ikey;
}
}
index[ikey] = offsets;
IncrementIndexSize(offsets.Count);
}
}
catch (Exception)
{
// done
// }
// finally
// {
if (streamtype == StreamFactory.StreamSecurityType.Secure)
{
iout.Position = 0;
IndexHash = hasher.ComputeHash(iout);
}
// Console.WriteLine("********************* closed " + IndexFQN);
index_br.Close();
iout.Close();
GC.Collect();
}
}
else
{
return(false);
}
}
catch (Exception exp)
{
Console.WriteLine("{0} Exception caught.", exp);
return(false);
}
return(true);
}