示例#1
0
        public new IValue Get(IKey key)
        {
            if (logger != null)
            {
                logger.Log("Start FileDataStream Retreive FilePathValue");
            }
            IValue        valuePath = base.Get(key, true);
            DataBlockInfo dbi       = base.GetDBI(key);

            if (logger != null)
            {
                logger.Log("End FileDataStream Retreive FilePathValue");
            }
            return(ReadData(valuePath, dbi));
        }
示例#2
0
        public Tuple <IValue, long> GetLatest(IKey tag)
        {
            Tuple <IValue, long> latest_tuple = null;

            if (tag != null)
            {
                IValue value = Get(tag);
                if (value != null)
                {
                    DataBlockInfo dbi = GetDBI(tag);
                    latest_tuple = new Tuple <IValue, long>(value, dbi.ts);
                }
            }
            return(latest_tuple);
        }
示例#3
0
        private List <DataBlockInfo> GetOffsetListFromHeader(IKey key)
        {
            if (!indexHeader.ContainsKey(key))
            {
                return(null);
            }

            long keyOffset = indexHeader[key];

            string IndexFilePath = targetDir + "/" + IndexFileName;

            if (!File.Exists(IndexFilePath))
            {
                throw new Exception("index file is missing!!!!! ");
            }

            FileStream   iout     = new FileStream(IndexFilePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
            BinaryReader index_br = new BinaryReader(iout);

            index_br.BaseStream.Seek(keyOffset, SeekOrigin.Begin);

            string keyInFile = index_br.ReadString();// increment file pointer
            //IKey ikey = SerializerHelper<KeyType>.DeserializeFromJsonStream(keyInFile) as IKey;
            // TODO compare with Ikey above


            int num_offsets = index_br.ReadInt32();
            List <DataBlockInfo> offsets = new List <DataBlockInfo>(num_offsets);

            for (int i = 0; i < num_offsets; i++)
            {
                long          ts     = index_br.ReadInt64();
                long          offset = index_br.ReadInt64();
                DataBlockInfo tso    = new DataBlockInfo(ts, offset);

                if (streamtype == StreamFactory.StreamSecurityType.Secure)
                {
                    Int32 h_len = index_br.ReadInt32();
                    if (h_len > 0)
                    {
                        tso.hashValue = index_br.ReadBytes(h_len);
                    }
                }
                offsets.Add(tso);
            }

            return(offsets);
        }
示例#4
0
        /// <summary>
        /// ValuePath is just the value file name ts.dat and not fully qualified file path
        /// </summary>
        /// <param name="valuePath"></param>
        /// <param name="dbi"></param>
        /// <returns></returns>
        internal ByteValue ReadData(IValue valuePath, DataBlockInfo dbi)
        {
            ByteValue byteValue   = null;
            string    FQValuePath = targetDir + "/" + valuePath;


            if (logger != null)
            {
                logger.Log("Start Synchronizer Simple Download");
            }
            if (null != valuePath && remoteRead)
            {
                if (!synchronizer.DownloadFile(valuePath.ToString(), FQValuePath))
                {
                    return(byteValue);
                }
            }
            if (logger != null)
            {
                logger.Log("End Synchronizer Simple Download");
            }

            if (logger != null)
            {
                logger.Log("Start FileDataStream ReadFromDisk");
            }
            if (null != valuePath)
            {
                FileStream fout = new FileStream(FQValuePath,
                                                 FileMode.OpenOrCreate,
                                                 FileAccess.Read,
                                                 FileShare.ReadWrite);
                fout.Seek(0, SeekOrigin.Begin);
                byte[] bytes = new byte[fout.Length];
                //read file to MemoryStream
                int bytesRead = 0;
                fout.Read(bytes, bytesRead, (int)fout.Length);
                fout.Close();
                byteValue = new ByteValue(bytes);
            }
            if (logger != null)
            {
                logger.Log("End FileDataStream ReadFromDisk");
            }
            if (streamtype == StreamFactory.StreamSecurityType.Secure)
            {
                if (logger != null)
                {
                    logger.Log("Start FileDataStream Decrypt DataBlock");
                }
                if (!hasher.ComputeHash(byteValue.GetBytes()).SequenceEqual(dbi.hashValue))
                {
                    return(null);
                }

                byteValue = new ByteValue(Crypto.DecryptBytesSimple(byteValue.GetBytes(), Crypto.KeyDer(acl_md.encKey), acl_md.IV));
                if (logger != null)
                {
                    logger.Log("End FileDataStream Decrypt DataBlock");
                }
            }
            return(byteValue);
        }
示例#5
0
        protected long UpdateHelper(IKey key, IValue value, bool IsAppend, Byte[] valHash = null, long timestamp = -1, long offsetInStream = -1)
        {
            long offset;

            if (!key.GetType().Equals(typeof(KeyType)))
            {
                throw new InvalidDataException("Invalid IKey Type");
            }
            if (!value.GetType().Equals(typeof(ValType)))
            {
                throw new InvalidDataException("Invalid IValue Type");
            }

            if (logger != null)
            {
                logger.Log("Start ValueDataStream Tag Lookup");
            }
            List <DataBlockInfo> offsets;

            // check if the entry is present (and record offset)
            if (index.ContainsKey(key))
            {
                // TODO: this could just have been a simple collision
                //   -- make sure that this is the same key
                //   -- if different, use a secondary hash function

                offsets = index[key];
            }
            else
            {
                offsets    = new List <DataBlockInfo>();
                index[key] = offsets;
                // IncrementIndexSize(offsets.Count);
                IncrementIndexSize(1);
            }
            if (logger != null)
            {
                logger.Log("End ValueDataStream Tag Lookup");
            }

            if (logger != null)
            {
                logger.Log("Start ValueDataStream Construct DataBlock");
            }
            // write <op (1B), ts, key, val_len, val>
            byte op;

            if (IsAppend)
            {
                op = (byte)WriteOp.AppendOp;
            }
            else
            {
                op = (byte)WriteOp.UpdateOp;
            }


            long ts;

            // Construct datablock
            if (timestamp == -1)
            {
                ts = StreamFactory.NowUtc();
            }
            else
            {
                ts = timestamp;
            }

            DataBlock <KeyType, ValType> db = new DataBlock <KeyType, ValType>();

            db.op = op;
            //db.timestamp = ts;
            //db.setKey(key);

            /*
             * Commenting out because the synchronizer will do this now at the chunk level
             * if (streamtype == StreamFactory.StreamSecurityType.Secure)
             * {
             *  //if (logger != null) logger.Log("Start encrypting value");
             *  db.value = new ByteValue(Crypto.EncryptBytesSimple(value.GetBytes(), Crypto.KeyDer(acl_md.encKey), acl_md.IV));
             *  //if (logger != null) logger.Log("End encrypting value");
             * }
             * else*/
            {
                //db.value = new ByteValue(value.GetBytes());
                db.setValue(value);
            }
            if (logger != null)
            {
                logger.Log("End ValueDataStream Construct DataBlock");
            }

            if (offsetInStream == -1)
            {
                if (logger != null)
                {
                    logger.Log("Start ValueDataStream Serialize DataBlock");
                }
                //Byte[] buffer = db.SerializeToByteStream().ToArray();
                Byte[] buffer = SerializerHelper <DataBlock <KeyType, ValType> > .SerializeToProtoStream(db).ToArray();

                if (logger != null)
                {
                    logger.Log("End ValueDataStream Serialize DataBlock");
                }

                if (logger != null)
                {
                    logger.Log("Start ValueDataStream WriteToDisc DataBlock");
                }
                // fs_bw.Write(db.SerializeToJsonStream());
                // get file offset; add <key_hash, offset> to index
                fs_bw.BaseStream.Seek(0, SeekOrigin.End);
                offset = fs_bw.BaseStream.Position;
                fs_bw.Write(buffer.Length);
                fs_bw.Write(buffer);
                fs_bw.Flush();
                if (logger != null)
                {
                    logger.Log("End ValueDataStream WriteToDisc DataBlock");
                }
            }
            else
            {
                offset = offsetInStream;
            }
            // Construct dbinfo in index
            if (logger != null)
            {
                logger.Log("Start ValueDataStream Construct and Add DataBlockInfo");
            }
            DataBlockInfo latest_tso = new DataBlockInfo(ts, offset);

            if (streamtype == StreamFactory.StreamSecurityType.Secure)
            {
                if (valHash != null) // passed from the dir stream and written into datablockinfo. not written for file stream
                {
                    latest_tso.hashValue = valHash;
                }

                /*
                 *  else
                 *  {
                 *      if (logger != null) logger.Log("Start taking hash of encrypted value");
                 *      latest_tso.hashValue = hasher.ComputeHash(db.value.GetBytes());
                 *      if (logger != null) logger.Log("End taking hash of encrypted value");
                 *
                 *  }
                 *   latest_tso.key_version = acl_md.keyVersion;
                 */
            }

            // apply to index
            if (IsAppend)
            {
                offsets.Add(latest_tso);
                // IncrementIndexSize();
            }
            else
            {
                if (offsets.Count == 0)
                {
                    offsets.Add(latest_tso);
                    // IncrementIndexSize();
                }
                else
                {
                    offsets[offsets.Count - 1] = latest_tso;
                }
            }
            if (logger != null)
            {
                logger.Log("End ValueDataStream Construct and Add DataBlockInfo");
            }
            return(offset);
        }
示例#6
0
        protected bool FillIndex()
        {
            try
            {
                string IndexFQN = targetDir + "/" + IndexFileName;
                //Console.WriteLine("***************** filling index : "+ IndexFQN);
                if (File.Exists(IndexFQN))
                {
                    FileStream iout = new FileStream(IndexFQN, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);

                    BinaryReader index_br = new BinaryReader(iout);

                    long latest_ts = 0;

                    try
                    {
                        while (true)
                        {
                            string key  = index_br.ReadString();
                            IKey   ikey = SerializerHelper <KeyType> .DeserializeFromJsonStream(key) as IKey;

                            int num_offsets = index_br.ReadInt32();
                            List <DataBlockInfo> offsets = new List <DataBlockInfo>(num_offsets);
                            for (int i = 0; i < num_offsets; i++)
                            {
                                long          ts     = index_br.ReadInt64();
                                long          offset = index_br.ReadInt64();
                                DataBlockInfo tso    = new DataBlockInfo(ts, offset);

                                if (streamtype == StreamFactory.StreamSecurityType.Secure)
                                {
                                    Int32 h_len = index_br.ReadInt32();
                                    if (h_len > 0)
                                    {
                                        tso.hashValue = index_br.ReadBytes(h_len);
                                    }
                                }
                                offsets.Add(tso);
                                if (ts > latest_ts)
                                {
                                    latest_ts  = ts;
                                    latest_key = ikey;
                                }
                            }
                            index[ikey] = offsets;
                            IncrementIndexSize(offsets.Count);
                        }
                    }
                    catch (Exception)
                    {
                        // done

                        //    }
                        //    finally
                        //      {
                        if (streamtype == StreamFactory.StreamSecurityType.Secure)
                        {
                            iout.Position = 0;
                            IndexHash     = hasher.ComputeHash(iout);
                        }

                        // Console.WriteLine("********************* closed " + IndexFQN);
                        index_br.Close();
                        iout.Close();
                        GC.Collect();
                    }
                }
                else
                {
                    return(false);
                }
            }
            catch (Exception exp)
            {
                Console.WriteLine("{0} Exception caught.", exp);
                return(false);
            }
            return(true);
        }