protected void Initialize(Logger Log, FqStreamID FQSID,
int num, CallerInfo Ci, LocationInfo Li,
StreamFactory.StreamOp op, StreamFactory.StreamSecurityType type,
CompressionType ctype, int ChunkSizeForUpload, int ThreadPoolSize)
{
// Logging related
logger = Log;
if (logger != null)
{
logger.Log("Start ValueDataStream Init DataStructures");
}
// Naming related
streamid = FQSID;
stream = new MetaDataService.FQStreamID();
stream.HomeId = FQSID.HomeId;
stream.AppId = FQSID.AppId;
stream.StreamId = FQSID.StreamId;
BaseDir = Path.GetFullPath((null != Ci.workingDir) ? Ci.workingDir : Directory.GetCurrentDirectory());
targetDir = BaseDir + "/" + streamid.ToString();
seq_num = num;
targetDir = targetDir + "/" + seq_num;
if (!Directory.Exists(targetDir))
{
Directory.CreateDirectory(targetDir);
}
// Ownership related
caller = new MetaDataService.Principal();
caller.HomeId = streamid.HomeId;
caller.AppId = Ci.appName;
owner = new MetaDataService.Principal();
owner.HomeId = streamid.HomeId;
owner.AppId = streamid.AppId;
// Location
account = new MetaDataService.AccountInfo();
account.accountName = Li.accountName;
account.accountKey = Li.accountKey;
account.location = "" + Li.st;
// Index related
index = new Dictionary <IKey, List <DataBlockInfo> >();
t_s = StreamFactory.NowUtc();
t_e = 0;
IndexHash = null;
isSealed = false;
// Low level
hasher = new SHA256CryptoServiceProvider();
// State maint
streamop = op;
streamtype = type;
streamcompressiontype = ctype;
//sync related
this.StreamChunkSizeForUpload = ChunkSizeForUpload;
this.StreamThreadPoolSize = ThreadPoolSize;
}
public OldDataDirStream(FqStreamID FQSID, StreamFactory.StreamOp Op, CallerInfo Ci, ISync sync)
: base(FQSID, Op, Ci, sync)
{
if (!typeof(IValue).IsAssignableFrom(typeof(ValType)))
{
throw new InvalidDataException("ValType must implement IValue");
}
}
public static string doDiskSpeed(string filesize, string blocksize, StreamFactory.StreamOp op)
{
// For the example
// Use ProcessStartInfo class
try
{
File.Delete("testfile.dat");
}
catch
{
}
ProcessStartInfo startInfo = new ProcessStartInfo();
startInfo.CreateNoWindow = false;
startInfo.UseShellExecute = false;
startInfo.FileName = "DiskSpd.exe";
startInfo.WindowStyle = ProcessWindowStyle.Hidden;
startInfo.Arguments = " -UN ";
if (op == StreamFactory.StreamOp.Write)
{
startInfo.Arguments += " -w ";
}
startInfo.Arguments += " -b" + blocksize;
startInfo.Arguments += " -c" + filesize;
startInfo.Arguments += " -d5";
startInfo.Arguments += " testfile.dat";
startInfo.RedirectStandardOutput = true;
string ret = "";
try
{
// Start the process with the info we specified.
// Call WaitForExit and then the using statement will close.
using (Process exeProcess = Process.Start(startInfo))
{
ret = exeProcess.StandardOutput.ReadToEnd();
exeProcess.WaitForExit();
}
}
catch (Exception e)
{
Console.WriteLine("Exception {0}", e);
// Log error.
}
return(ret);
}
public FileDataStream(Logger Log, FqStreamID FQSID, int num,
CallerInfo Ci, LocationInfo Li,
StreamFactory.StreamOp op, StreamFactory.StreamSecurityType type,
CompressionType ctype, int ChunkSizeForUpload, int UploadThreadPoolSize,
string prkey, string pukey, MetaDataService.ACLEntry key_md,
IndexInfo ii, bool alreadyExists = true)
: base(Log, FQSID, num, Ci, Li, op, type, ctype, ChunkSizeForUpload, UploadThreadPoolSize,
prkey, pukey, key_md, ii, alreadyExists: alreadyExists)
{
remoteRead = false;
// if remote stream, read op. fetch valueFile when needed for get queries
if (this.synchronizer != null && streamop == StreamFactory.StreamOp.Read)
{
remoteRead = true;
}
}
public ValueDataStream(Logger Log, FqStreamID FQSID, int num,
CallerInfo Ci, LocationInfo Li,
StreamFactory.StreamOp op, StreamFactory.StreamSecurityType type,
CompressionType ctype, int ChunkSizeForUpload, int UploadThreadPoolSize,
string prkey, string pukey, MetaDataService.ACLEntry key_md,
IndexInfo ii,
bool alreadyExists = true)
{
Initialize(Log, FQSID, num, Ci, Li, op, type, ctype, ChunkSizeForUpload, UploadThreadPoolSize);
if (streamtype == StreamFactory.StreamSecurityType.Secure)
{
// Key related
OwnerPriKey = prkey;
OwnerPubKey = pukey;
acl_md = key_md;
}
if (logger != null)
{
logger.Log("End ValueDataStream Init DataStructures");
}
// Fetch data
FetchAndFillIndex(ii, alreadyExists);
// Reset sync to upload
if (op == StreamFactory.StreamOp.Write)
{
CreateSync(SynchronizeDirection.Upload);
}
isClosed = false;
if (logger != null)
{
logger.Log("Start ValueDataStream ReadFromDisk Open");
}
OpenStream();
if (logger != null)
{
logger.Log("End ValueDataStream ReadFromDisk Open");
}
disposed = false;
}
/* syncIntervalSec:
* -ve ==> don't sync on writes; only sync on close.
* 0 ==> sync on every write
* +ve ==> sync every x seconds
*
* Throws System.Exception e.g., on network disconnection for remote streams. Catch in Caller.
*/
public IStream openFileDataStream <KeyType>(FqStreamID FQSID,
CallerInfo Ci,
LocationInfo Li,
StreamFactory.StreamSecurityType type,
CompressionType ctype,
StreamFactory.StreamOp op,
string mdserveraddress = null,
int ChunkSizeForUpload = 4 *1024 *1024,
int ThreadPoolSize = 1,
Logger log = null,
bool sideload = false,
int syncIntervalSec = -1)
where KeyType : IKey, new()
{
if (Li == null)
{
Li = new LocationInfo("", "", SynchronizerType.None);
}
return(new MetaStream <KeyType, ByteValue>(FQSID, Ci, Li,
op, type, ctype, StreamDataType.Files,
syncIntervalSec, mdserveraddress,
ChunkSizeForUpload, ThreadPoolSize,
log, sideload));
}
public OldDataFileStream(FqStreamID FQSID, StreamFactory.StreamOp Op, CallerInfo Ci, ISync sync)
{
if (!typeof(IKey).IsAssignableFrom(typeof(KeyType)))
{
throw new InvalidDataException("KeyType must implement IKey");
}
if (!typeof(IValue).IsAssignableFrom(typeof(ValType)))
{
throw new InvalidDataException("ValType must implement IValue");
}
callerId = Ci.friendlyName;
callerSecret = Ci.secret;
synchronizer = sync;
isClosed = false;
disposed = false;
sha1 = new SHA1CryptoServiceProvider();
index = new Dictionary <IKey, List <TS_Offset> >();
/* ts_index = new List<TS_Offset>(); */
latest_tso = new TS_Offset(0, 0);
// Get the current directory.
string BaseDir = Path.GetFullPath((null != Ci.workingDir) ? Ci.workingDir : Directory.GetCurrentDirectory());
targetDir = BaseDir + "/" + FQSID.ToString();
if (!Directory.Exists(targetDir))
{
Directory.CreateDirectory(targetDir);
}
if (synchronizer != null)
{
synchronizer.SetLocalSource(targetDir);
}
md = new OldMetaData(targetDir, ".md");
// Check if stream has to be CREATED
if (!md.load)
{
if (FQSID.AppId == callerId)
{
md.setOwner(FQSID.AppId);
md.SetReadAccess(FQSID.AppId);
md.SetWriteAccess(FQSID.AppId);
md.FlushMetaData();
Console.WriteLine("Created stream " + targetDir + " for " + callerId);
}
else
{
throw new InvalidOperationException(callerId + " not permitted to create stream for " + FQSID.AppId);
}
}
// Open stream for read or write
if (Op == StreamFactory.StreamOp.Read)
{
if (!OpenForRead())
{
throw new InvalidDataException("Couldn't open stream for reading");
}
}
else
{
if (!OpenForWrite())
{
throw new InvalidDataException("Couldn't open stream for writing");
}
}
// Build index
try
{
// permission checks succeeded
// load index from file if present
// TODO: if not and stream.dat is present: recreate index from stream.dat
string IndexFQN = targetDir + "/index.dat";
if (File.Exists(IndexFQN))
{
FileStream iout = new FileStream(IndexFQN, FileMode.Open,
FileAccess.Read, FileShare.ReadWrite);
BinaryReader index_br = new BinaryReader(iout);
try
{
while (true)
{
string key = index_br.ReadString();
IKey ikey = SerializerHelper <KeyType> .DeserializeFromJsonStream(key) as IKey;
int num_offsets = index_br.ReadInt32();
List <TS_Offset> offsets = new List <TS_Offset>(num_offsets);
for (int i = 0; i < num_offsets; i++)
{
long ts = index_br.ReadInt64();
long offset = index_br.ReadInt64();
TS_Offset tso = new TS_Offset(ts, offset);
offsets.Add(tso);
if (ts > latest_tso.ts)
{
latest_tso = tso;
}
}
index[ikey] = offsets;
}
;
}
catch (EndOfStreamException)
{
// done
}
finally
{
index_br.Close();
}
}
/*
* // load ts_index
* string TsIndexFQN = targetDir + "/ts_index.dat";
* if (File.Exists(TsIndexFQN))
* {
* FileStream iout = new FileStream(TsIndexFQN, FileMode.Open,
* FileAccess.Read, FileShare.ReadWrite);
*
* BinaryReader index_br = new BinaryReader(iout);
*
* try
* {
* while (true)
* {
* long ts = index_br.ReadInt64();
* long offset = index_br.ReadInt64();
* ts_index.Add(new TS_Offset(ts, offset));
* };
* }
* catch (EndOfStreamException)
* {
* // done
* }
* finally
* {
* index_br.Close();
* }
* }
*/
// create the FileStream
fout = new FileStream(targetDir + "/stream.dat", FileMode.OpenOrCreate,
FileAccess.Write, FileShare.ReadWrite);
fout.Seek(0, SeekOrigin.End);
fs_bw = new BinaryWriter(fout);
fin = new FileStream(targetDir + "/stream.dat", FileMode.Open,
FileAccess.Read, FileShare.ReadWrite);
fs_br = new BinaryReader(fin);
}
catch (Exception e)
{
Console.WriteLine("Failed to open file: " + targetDir + "/stream.dat");
Console.WriteLine("{0} Exception caught.", e);
}
}
