protected void Initialize(Logger Log, FqStreamID FQSID, int num, CallerInfo Ci, LocationInfo Li, StreamFactory.StreamOp op, StreamFactory.StreamSecurityType type, CompressionType ctype, int ChunkSizeForUpload, int ThreadPoolSize) { // Logging related logger = Log; if (logger != null) { logger.Log("Start ValueDataStream Init DataStructures"); } // Naming related streamid = FQSID; stream = new MetaDataService.FQStreamID(); stream.HomeId = FQSID.HomeId; stream.AppId = FQSID.AppId; stream.StreamId = FQSID.StreamId; BaseDir = Path.GetFullPath((null != Ci.workingDir) ? Ci.workingDir : Directory.GetCurrentDirectory()); targetDir = BaseDir + "/" + streamid.ToString(); seq_num = num; targetDir = targetDir + "/" + seq_num; if (!Directory.Exists(targetDir)) { Directory.CreateDirectory(targetDir); } // Ownership related caller = new MetaDataService.Principal(); caller.HomeId = streamid.HomeId; caller.AppId = Ci.appName; owner = new MetaDataService.Principal(); owner.HomeId = streamid.HomeId; owner.AppId = streamid.AppId; // Location account = new MetaDataService.AccountInfo(); account.accountName = Li.accountName; account.accountKey = Li.accountKey; account.location = "" + Li.st; // Index related index = new Dictionary <IKey, List <DataBlockInfo> >(); t_s = StreamFactory.NowUtc(); t_e = 0; IndexHash = null; isSealed = false; // Low level hasher = new SHA256CryptoServiceProvider(); // State maint streamop = op; streamtype = type; streamcompressiontype = ctype; //sync related this.StreamChunkSizeForUpload = ChunkSizeForUpload; this.StreamThreadPoolSize = ThreadPoolSize; }
public OldDataDirStream(FqStreamID FQSID, StreamFactory.StreamOp Op, CallerInfo Ci, ISync sync) : base(FQSID, Op, Ci, sync) { if (!typeof(IValue).IsAssignableFrom(typeof(ValType))) { throw new InvalidDataException("ValType must implement IValue"); } }
public static string doDiskSpeed(string filesize, string blocksize, StreamFactory.StreamOp op) { // For the example // Use ProcessStartInfo class try { File.Delete("testfile.dat"); } catch { } ProcessStartInfo startInfo = new ProcessStartInfo(); startInfo.CreateNoWindow = false; startInfo.UseShellExecute = false; startInfo.FileName = "DiskSpd.exe"; startInfo.WindowStyle = ProcessWindowStyle.Hidden; startInfo.Arguments = " -UN "; if (op == StreamFactory.StreamOp.Write) { startInfo.Arguments += " -w "; } startInfo.Arguments += " -b" + blocksize; startInfo.Arguments += " -c" + filesize; startInfo.Arguments += " -d5"; startInfo.Arguments += " testfile.dat"; startInfo.RedirectStandardOutput = true; string ret = ""; try { // Start the process with the info we specified. // Call WaitForExit and then the using statement will close. using (Process exeProcess = Process.Start(startInfo)) { ret = exeProcess.StandardOutput.ReadToEnd(); exeProcess.WaitForExit(); } } catch (Exception e) { Console.WriteLine("Exception {0}", e); // Log error. } return(ret); }
public FileDataStream(Logger Log, FqStreamID FQSID, int num, CallerInfo Ci, LocationInfo Li, StreamFactory.StreamOp op, StreamFactory.StreamSecurityType type, CompressionType ctype, int ChunkSizeForUpload, int UploadThreadPoolSize, string prkey, string pukey, MetaDataService.ACLEntry key_md, IndexInfo ii, bool alreadyExists = true) : base(Log, FQSID, num, Ci, Li, op, type, ctype, ChunkSizeForUpload, UploadThreadPoolSize, prkey, pukey, key_md, ii, alreadyExists: alreadyExists) { remoteRead = false; // if remote stream, read op. fetch valueFile when needed for get queries if (this.synchronizer != null && streamop == StreamFactory.StreamOp.Read) { remoteRead = true; } }
public ValueDataStream(Logger Log, FqStreamID FQSID, int num, CallerInfo Ci, LocationInfo Li, StreamFactory.StreamOp op, StreamFactory.StreamSecurityType type, CompressionType ctype, int ChunkSizeForUpload, int UploadThreadPoolSize, string prkey, string pukey, MetaDataService.ACLEntry key_md, IndexInfo ii, bool alreadyExists = true) { Initialize(Log, FQSID, num, Ci, Li, op, type, ctype, ChunkSizeForUpload, UploadThreadPoolSize); if (streamtype == StreamFactory.StreamSecurityType.Secure) { // Key related OwnerPriKey = prkey; OwnerPubKey = pukey; acl_md = key_md; } if (logger != null) { logger.Log("End ValueDataStream Init DataStructures"); } // Fetch data FetchAndFillIndex(ii, alreadyExists); // Reset sync to upload if (op == StreamFactory.StreamOp.Write) { CreateSync(SynchronizeDirection.Upload); } isClosed = false; if (logger != null) { logger.Log("Start ValueDataStream ReadFromDisk Open"); } OpenStream(); if (logger != null) { logger.Log("End ValueDataStream ReadFromDisk Open"); } disposed = false; }
/* syncIntervalSec: * -ve ==> don't sync on writes; only sync on close. * 0 ==> sync on every write * +ve ==> sync every x seconds * * Throws System.Exception e.g., on network disconnection for remote streams. Catch in Caller. */ public IStream openFileDataStream <KeyType>(FqStreamID FQSID, CallerInfo Ci, LocationInfo Li, StreamFactory.StreamSecurityType type, CompressionType ctype, StreamFactory.StreamOp op, string mdserveraddress = null, int ChunkSizeForUpload = 4 *1024 *1024, int ThreadPoolSize = 1, Logger log = null, bool sideload = false, int syncIntervalSec = -1) where KeyType : IKey, new() { if (Li == null) { Li = new LocationInfo("", "", SynchronizerType.None); } return(new MetaStream <KeyType, ByteValue>(FQSID, Ci, Li, op, type, ctype, StreamDataType.Files, syncIntervalSec, mdserveraddress, ChunkSizeForUpload, ThreadPoolSize, log, sideload)); }
public OldDataFileStream(FqStreamID FQSID, StreamFactory.StreamOp Op, CallerInfo Ci, ISync sync) { if (!typeof(IKey).IsAssignableFrom(typeof(KeyType))) { throw new InvalidDataException("KeyType must implement IKey"); } if (!typeof(IValue).IsAssignableFrom(typeof(ValType))) { throw new InvalidDataException("ValType must implement IValue"); } callerId = Ci.friendlyName; callerSecret = Ci.secret; synchronizer = sync; isClosed = false; disposed = false; sha1 = new SHA1CryptoServiceProvider(); index = new Dictionary <IKey, List <TS_Offset> >(); /* ts_index = new List<TS_Offset>(); */ latest_tso = new TS_Offset(0, 0); // Get the current directory. string BaseDir = Path.GetFullPath((null != Ci.workingDir) ? Ci.workingDir : Directory.GetCurrentDirectory()); targetDir = BaseDir + "/" + FQSID.ToString(); if (!Directory.Exists(targetDir)) { Directory.CreateDirectory(targetDir); } if (synchronizer != null) { synchronizer.SetLocalSource(targetDir); } md = new OldMetaData(targetDir, ".md"); // Check if stream has to be CREATED if (!md.load) { if (FQSID.AppId == callerId) { md.setOwner(FQSID.AppId); md.SetReadAccess(FQSID.AppId); md.SetWriteAccess(FQSID.AppId); md.FlushMetaData(); Console.WriteLine("Created stream " + targetDir + " for " + callerId); } else { throw new InvalidOperationException(callerId + " not permitted to create stream for " + FQSID.AppId); } } // Open stream for read or write if (Op == StreamFactory.StreamOp.Read) { if (!OpenForRead()) { throw new InvalidDataException("Couldn't open stream for reading"); } } else { if (!OpenForWrite()) { throw new InvalidDataException("Couldn't open stream for writing"); } } // Build index try { // permission checks succeeded // load index from file if present // TODO: if not and stream.dat is present: recreate index from stream.dat string IndexFQN = targetDir + "/index.dat"; if (File.Exists(IndexFQN)) { FileStream iout = new FileStream(IndexFQN, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); BinaryReader index_br = new BinaryReader(iout); try { while (true) { string key = index_br.ReadString(); IKey ikey = SerializerHelper <KeyType> .DeserializeFromJsonStream(key) as IKey; int num_offsets = index_br.ReadInt32(); List <TS_Offset> offsets = new List <TS_Offset>(num_offsets); for (int i = 0; i < num_offsets; i++) { long ts = index_br.ReadInt64(); long offset = index_br.ReadInt64(); TS_Offset tso = new TS_Offset(ts, offset); offsets.Add(tso); if (ts > latest_tso.ts) { latest_tso = tso; } } index[ikey] = offsets; } ; } catch (EndOfStreamException) { // done } finally { index_br.Close(); } } /* * // load ts_index * string TsIndexFQN = targetDir + "/ts_index.dat"; * if (File.Exists(TsIndexFQN)) * { * FileStream iout = new FileStream(TsIndexFQN, FileMode.Open, * FileAccess.Read, FileShare.ReadWrite); * * BinaryReader index_br = new BinaryReader(iout); * * try * { * while (true) * { * long ts = index_br.ReadInt64(); * long offset = index_br.ReadInt64(); * ts_index.Add(new TS_Offset(ts, offset)); * }; * } * catch (EndOfStreamException) * { * // done * } * finally * { * index_br.Close(); * } * } */ // create the FileStream fout = new FileStream(targetDir + "/stream.dat", FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite); fout.Seek(0, SeekOrigin.End); fs_bw = new BinaryWriter(fout); fin = new FileStream(targetDir + "/stream.dat", FileMode.Open, FileAccess.Read, FileShare.ReadWrite); fs_br = new BinaryReader(fin); } catch (Exception e) { Console.WriteLine("Failed to open file: " + targetDir + "/stream.dat"); Console.WriteLine("{0} Exception caught.", e); } }