public Packet ReadPacket(Stream stream)
{
int readValue = stream.ReadByte();
if (readValue == -1)
{
return(null);
}
byte transferId = (byte)readValue;
int?transferLengthNullable = StreamOperation.ReadShort(stream);
if (transferLengthNullable == null)
{
return(null);
}
int transferLength = (int)transferLengthNullable;
short?dataLengthNullable = StreamOperation.ReadShort(stream);
if (dataLengthNullable == null)
{
return(null);
}
short dataLength = (short)dataLengthNullable;
byte[] data = StreamOperation.ReadBytes(stream, dataLength);
if (data == null)
{
return(null);
}
return(new NewTransferPacket(transferId, transferLength, data));
}
public StreamMarket ToStream(StreamOperation operation)
{
return(new StreamMarket()
{
Id = Id,
Name = Name,
Operation = operation
});
}
public override int Read(byte[] buffer, int offset, int count)
{
if (_lastOperation != StreamOperation.Read)
{
Seek(0, SeekOrigin.Begin);
_lastOperation = StreamOperation.Read;
}
return(base.Read(buffer, offset, count));
}
public override void Write(byte[] buffer, int offset, int count)
{
if (_lastOperation != StreamOperation.Write)
{
Seek(0, SeekOrigin.Begin);
_lastOperation = StreamOperation.Write;
}
base.Write(buffer, offset, count);
}
public StreamInstrument ToStream(StreamOperation operation)
{
return(new StreamInstrument()
{
Id = Id,
SubmarketId = SubmarketId,
Name = Name,
Operation = operation
});
}
public StreamRule ToStream(StreamOperation operation)
{
return(new StreamRule()
{
Id = Id,
PortfolioId = PortfolioId,
Expression = Expression,
Operation = operation
});
}
private IEnumerator <TimeSeriesStreamResult <T> > StreamTimeSeriesInternal <T>(IDocumentQuery <T> query) where T : ITimeSeriesQueryStreamResult, new()
{
var streamOperation = new StreamOperation(this);
var command = streamOperation.CreateRequest(query.GetIndexQuery());
RequestExecutor.Execute(command, Context, sessionInfo: _sessionInfo);
var result = streamOperation.SetResultForTimeSeries(command.Result);
return(YieldTimeSeriesResults(query, result));
}
public IEnumerator <StreamResult <T> > Stream <T>(IDocumentQuery <T> query)
{
var streamOperation = new StreamOperation(this);
var command = streamOperation.CreateRequest(query.GetIndexQuery());
RequestExecutor.Execute(command, Context, sessionInfo: _sessionInfo);
var result = streamOperation.SetResult(command.Result);
return(YieldResults(query, result));
}
public StreamPrice ToPriceStream(StreamOperation operation)
{
return(new StreamPrice()
{
Id = "Price:" + Id.Substring(11),
//Id = Guid.NewGuid().ToString(),
PriceLatest = PriceLatest,
PriceDate = PriceDate,
Operation = operation
});
}
public StreamPortfolio ToStream(StreamOperation operation)
{
return new StreamPortfolio()
{
Id = Id,
Name = Name,
Balance = Balance,
CheckRules = CheckRules,
Operation = operation
};
}
public async Task <IAsyncEnumerator <StreamResult <T> > > StreamAsync <T>(string startsWith, string matches = null, int start = 0,
int pageSize = int.MaxValue, string startAfter = null, CancellationToken token = default)
{
var streamOperation = new StreamOperation(this);
var command = streamOperation.CreateRequest(startsWith, matches, start, pageSize, null, startAfter);
await RequestExecutor.ExecuteAsync(command, Context, SessionInfo, token).ConfigureAwait(false);
var result = streamOperation.SetResultAsync(command.Result);
return(new YieldStream <T>(this, null, null, result, token));
}
public StreamPosition ToStream(StreamOperation operation)
{
return(new StreamPosition()
{
Id = Id,
PortfolioId = PortfolioId,
InstrumentId = InstrumentId,
Price = Price,
Volume = Volume,
Operation = operation
});
}
public async Task StreamIntoAsync <T>(IAsyncDocumentQuery <T> query, Stream output, CancellationToken token = default)
{
var streamOperation = new StreamOperation(this);
var command = streamOperation.CreateRequest(query.GetIndexQuery());
await RequestExecutor.ExecuteAsync(command, Context, SessionInfo, token).ConfigureAwait(false);
using (command.Result.Response)
using (command.Result.Stream)
{
await command.Result.Stream.CopyToAsync(output, 16 * 1024, token).ConfigureAwait(false);
}
}
public async Task <IAsyncEnumerator <StreamResult <T> > > StreamAsync <T>(IAsyncDocumentQuery <T> query, CancellationToken token = default(CancellationToken))
{
var streamOperation = new StreamOperation(this);
var command = streamOperation.CreateRequest((IRavenQueryInspector)query);
await RequestExecuter.ExecuteAsync(command, Context, token).ConfigureAwait(false);
var result = streamOperation.SetResultAsync(command.Result);
var queryOperation = ((AsyncDocumentQuery <T>)query).InitializeQueryOperation();
queryOperation.DisableEntitiesTracking = true;
return(new YieldStream <T>(this, result, token));
}
public void StreamInto <T>(IDocumentQuery <T> query, Stream output)
{
var streamOperation = new StreamOperation(this);
var command = streamOperation.CreateRequest(query.GetIndexQuery());
RequestExecutor.Execute(command, Context, sessionInfo: _sessionInfo);
using (command.Result.Response)
using (command.Result.Stream)
{
command.Result.Stream.CopyTo(output);
}
}
private async Task <IAsyncEnumerator <StreamResult <T> > > StreamAsync <T>(long?fromEtag, string startsWith, string matches,
int start, int pageSize, RavenPagingInformation pagingInformation = null, string skipAfter = null, string transformer = null,
Dictionary <string, object> transformerParameters = null, CancellationToken token = default(CancellationToken))
{
var streamOperation = new StreamOperation(this);
var command = streamOperation.CreateRequest(fromEtag, startsWith, matches, start, pageSize, null, pagingInformation, skipAfter, transformer,
transformerParameters);
await RequestExecuter.ExecuteAsync(command, Context, token).ConfigureAwait(false);
var result = streamOperation.SetResultAsync(command.Result);
return(new YieldStream <T>(this, result, token));
}
public IEnumerator <StreamResult <T> > Stream <T>(IDocumentQuery <T> query, out StreamQueryStatistics streamQueryStats)
{
var stats = new StreamQueryStatistics();
var streamOperation = new StreamOperation(this, stats);
var command = streamOperation.CreateRequest(query.GetIndexQuery());
RequestExecutor.Execute(command, Context, sessionInfo: SessionInfo);
using (var result = streamOperation.SetResult(command.Result))
{
streamQueryStats = stats;
return(YieldResults(query, result));
}
}
private ConsoleColor GetConsoleColor(StreamOperation operation)
{
switch (operation)
{
case StreamOperation.Read:
return(this.readColor);
case StreamOperation.Write:
return(this.writeColor);
default:
throw new UnhandledEnumException(operation);
}
}
private IEnumerator <TimeSeriesStreamResult <T> > StreamTimeSeriesInternal <T>(IDocumentQuery <T> query, out StreamQueryStatistics streamQueryStats) where T : ITimeSeriesQueryStreamResult, new()
{
var stats = new StreamQueryStatistics();
var streamOperation = new StreamOperation(this, stats);
var command = streamOperation.CreateRequest(query.GetIndexQuery());
RequestExecutor.Execute(command, Context, sessionInfo: _sessionInfo);
streamOperation.EnsureIsAcceptable(query.IndexName, command.Result);
var result = streamOperation.SetResultForTimeSeries(command.Result);
streamQueryStats = stats;
return(YieldTimeSeriesResults(query, result));
}
public StreamTrade ToStream(StreamOperation operation)
{
return(new StreamTrade()
{
Id = Id,
PortfolioId = PortfolioId,
InstrumentId = InstrumentId,
Bid = Bid,
Price = Price,
Volume = Volume,
Buyer = Buyer,
Seller = Seller,
Operation = operation
});
}
public List <string> Dump(string CallerName,
string HomeName, string AppName, string StreamName,
StreamType stream_type,
StreamOperation stream_op,
StreamFactory.StreamDataType ptype,
Byte[] value, int num_operations,
SynchronizerType synctype,
bool doCosts = false)
{
List <string> exp_details = new List <string>();
exp_details.Add(exp_id);
File.AppendAllLines(exp_directory + "/exp", exp_details);
return(exp_details);
}
public IEnumerator <StreamResult <T> > Stream <T>(IDocumentQuery <T> query)
{
var streamOperation = new StreamOperation(this);
var command = streamOperation.CreateRequest((IRavenQueryInspector)query);
RequestExecuter.Execute(command, Context);
using (var result = streamOperation.SetResult(command.Result))
{
while (result.MoveNext())
{
var res = result.Current;
var stremResult = CreateStreamResult <T>(res);
yield return(stremResult);
}
}
}
public async Task <IAsyncEnumerator <StreamResult <T> > > StreamAsync <T>(IAsyncDocumentQuery <T> query, CancellationToken token = default(CancellationToken))
{
var documentQuery = (AsyncDocumentQuery <T>)query;
var projectionFields = documentQuery.FieldsToFetchToken?.Projections;
var indexQuery = query.GetIndexQuery();
var streamOperation = new StreamOperation(this);
var command = streamOperation.CreateRequest(indexQuery);
await RequestExecutor.ExecuteAsync(command, Context, token, SessionInfo).ConfigureAwait(false);
var result = streamOperation.SetResultAsync(command.Result);
var queryOperation = ((AsyncDocumentQuery <T>)query).InitializeQueryOperation();
queryOperation.DisableEntitiesTracking = true;
return(new YieldStream <T>(this, query, projectionFields, result, token));
}
private async Task <IAsyncEnumerator <StreamResult <T> > > PerformQueryStreamOperation <T>(IAsyncDocumentQuery <T> query, StreamQueryStatistics streamQueryStats, CancellationToken token)
{
var documentQuery = (AsyncDocumentQuery <T>)query;
var fieldsToFetch = documentQuery.FieldsToFetchToken;
var indexQuery = query.GetIndexQuery();
var streamOperation = new StreamOperation(this, streamQueryStats);
var command = streamOperation.CreateRequest(indexQuery);
await RequestExecutor.ExecuteAsync(command, Context, SessionInfo, token).ConfigureAwait(false);
var result = streamOperation.SetResultAsync(command.Result);
var queryOperation = ((AsyncDocumentQuery <T>)query).InitializeQueryOperation();
queryOperation.NoTracking = true;
return(new YieldStream <T>(this, query, fieldsToFetch, result, token));
}
/// <summary>
/// Called whenever we've subscribed to monitor a payload in a streaming fashion. This is
/// useful for say, virus scanning without forcing the entire payload to be buffered into
/// memory before it is streamed to the user, or to monitor and decode video on the fly
/// without affecting the user. You can terminate the stream at any time while monitoring.
/// </summary>
/// <param name="messageInfo">
/// The originating http message item.
/// </param>
/// <param name="operation">
/// The operation kind.
/// </param>
/// <param name="buffer">
/// The data that passed through the stream.
/// </param>
/// <param name="dropConnection">
/// Whether or not to immediately terminate the connection.
/// </param>
private static void OnStreamedContentInspection(HttpMessageInfo messageInfo, StreamOperation operation, Memory <byte> buffer, out bool dropConnection)
{
var toFrom = operation == StreamOperation.Read ? "from" : "to";
Console.WriteLine($"Stream {operation} {buffer.Length} bytes {toFrom} {messageInfo.Url}");
dropConnection = false;
// Drop googlevideo.com videos.
if (messageInfo.Url.Host.IndexOf(".googlevideo.com") > -1)
{
// This basically means you can't watch anything on youtube. You can still load the
// site, but you can't play any videos.
// This is just to demonstrate that it's possible to have complete
// control over unbuffered streams.
// dropConnection = true;
}
}
private async Task <IAsyncEnumerator <TimeSeriesStreamResult <T> > > TimeSeriesStreamInternalAsync <T>(IAsyncDocumentQuery <T> query, StreamQueryStatistics streamQueryStats, CancellationToken token) where T : ITimeSeriesQueryStreamResult, new()
{
using (AsyncTaskHolder())
{
var documentQuery = (AsyncDocumentQuery <T>)query;
var indexQuery = query.GetIndexQuery();
var streamOperation = new StreamOperation(this, streamQueryStats);
var command = streamOperation.CreateRequest(indexQuery);
await RequestExecutor.ExecuteAsync(command, Context, _sessionInfo, token).ConfigureAwait(false);
var result = await streamOperation.SetResultForTimeSeriesAsync(command.Result).ConfigureAwait(false);
var queryOperation = ((AsyncDocumentQuery <T>)query).InitializeQueryOperation();
queryOperation.NoTracking = true;
return(new YieldTimeSeriesStream <T>(this, documentQuery, result, token));
}
}
private IEnumerator <StreamResult <T> > Stream <T>(long?fromEtag, string startsWith, string matches, int start, int pageSize, RavenPagingInformation pagingInformation,
string skipAfter, string transformer, Dictionary <string, object> transformerParameters)
{
var streamOperation = new StreamOperation(this);
var command = streamOperation.CreateRequest(fromEtag, startsWith, matches, start, pageSize, null, pagingInformation, skipAfter, transformer,
transformerParameters);
RequestExecuter.Execute(command, Context);
using (var result = streamOperation.SetResult(command.Result))
{
while (result.MoveNext())
{
var res = result.Current;
var stremResult = CreateStreamResult <T>(res);
yield return(stremResult);
}
}
}
public IEnumerator <StreamResult <T> > Stream <T>(string startsWith, string matches = null, int start = 0, int pageSize = int.MaxValue,
string startAfter = null)
{
var streamOperation = new StreamOperation(this);
var command = streamOperation.CreateRequest(startsWith, matches, start, pageSize, null, startAfter);
RequestExecutor.Execute(command, Context, sessionInfo: SessionInfo);
using (var result = streamOperation.SetResult(command.Result))
{
while (result.MoveNext())
{
var json = result.Current;
yield return(CreateStreamResult <T>(json, null));
}
}
}
/// <summary>
/// Attempts to complete the current operation. This is atomic to <see cref="StreamOperation.None"/>.
/// </summary>
private void CompleteOperation(StreamOperationToken token)
{
Analysis.IgnoreArgument(token);
lock (m_operationLock)
{
if (m_currentOperation == StreamOperation.None)
{
Contract.Assume(false, "Attempted to complete an operation, but no operation was in progress");
}
// Dispose is terminal.
if (m_currentOperation != StreamOperation.Dispose)
{
m_currentOperation = StreamOperation.None;
}
}
}
/// <summary>
/// Attempts to transition into a new operation. This is atomic from <see cref="StreamOperation.None"/>.
/// </summary>
protected StreamOperationToken StartNewOperation(StreamOperation next)
{
Contract.Requires(next != StreamOperation.None);
lock (m_operationLock)
{
if (m_currentOperation != StreamOperation.None)
{
Contract.Assume(
false,
I($"Attempted to start a {next:G} operation, but a {m_currentOperation:G} operation is already in progress on the same stream"));
}
m_currentOperation = next;
}
return(new StreamOperationToken(this));
}
public static Logger doDiskRaw(StreamOperation op,
int num_operations, int val_size,
StreamFactory.StreamPhysicalType ptype,
string exp_dir)
{
Logger logger = new Logger();
if (op == StreamOperation.RandomKeyRandomValueAppend ||
op == StreamOperation.SameKeyRandomValueAppend ||
op == StreamOperation.RandomKeySameValueAppend ||
op == StreamOperation.SameKeySameValueAppend)
{
// Populate the keys and the values
Random random = new Random(DateTime.Now.Millisecond);
StrKey[] keys = new StrKey[num_operations];
for (int i = 0; i < num_operations; ++i)
{
keys[i] = new StrKey("" + i);
}
Byte[] singleval = new Byte[val_size];
random.NextBytes(singleval);
Byte[][] tmp = new Byte[num_operations][];
if (!(op == StreamOperation.RandomKeySameValueAppend ||
op == StreamOperation.SameKeySameValueAppend))
{
for (int i = 0; i < num_operations; ++i)
{
tmp[i] = new Byte[val_size];
random.NextBytes(tmp[i]);
}
}
StrKey key = new StrKey("ExpKey");
if (ptype == StreamFactory.StreamPhysicalType.File)
{
// ValueDataStream type
logger.Log("Start Stream Open");
FileStream fout = new FileStream(exp_dir + "/DiskRaw", FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite);
fout.Seek(0, SeekOrigin.End);
BinaryWriter fs_bw = new BinaryWriter(fout);
logger.Log("End Stream Open");
// TimeSpan span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0));
// Console.WriteLine("Time" + span.TotalSeconds);
logger.Log("Start Stream Append");
for (int i = 0; i < num_operations; ++i)
{
fs_bw.BaseStream.Seek(0, SeekOrigin.End);
/*
if (op == StreamOperation.SameKeySameValueAppend || op == StreamOperation.SameKeyRandomValueAppend)
{
fs_bw.Write(key.key);
}
else
{
*/
fs_bw.Write(keys[i].key);
/*
}
*/
fs_bw.Write(StreamFactory.NowUtc());
/*
if (!(op == StreamOperation.RandomKeySameValueAppend ||
op == StreamOperation.SameKeySameValueAppend))
fs_bw.Write(tmp[i]);
else
*/
fs_bw.Write(singleval);
fs_bw.Flush();
}
fout.Flush(true);
fs_bw.Close();
// span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0));
// Console.WriteLine("Time" + span.TotalSeconds);
logger.Log("End Stream Append");
}
else
{
string[] filenames = new string[num_operations];
for (int i = 0; i < num_operations; ++i)
{
filenames[i] = exp_dir + "/DiskRaw" + i;
}
// FileDataStream type
logger.Log("Start Stream Append");
for (int i = 0; i < num_operations; ++i)
{
FileStream fout = new FileStream(filenames[i], FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite);
// fout.Seek(0, SeekOrigin.End);
BinaryWriter fs_bw = new BinaryWriter(fout);
// fs_bw.BaseStream.Seek(0, SeekOrigin.End);
/*
if (op == StreamOperation.SameKeySameValueAppend || op == StreamOperation.SameKeyRandomValueAppend)
{
fs_bw.Write(key.key);
}
else
{
*/
fs_bw.Write(keys[i].key);
// }
fs_bw.Write(StreamFactory.NowUtc());
/*
if (!(op == StreamOperation.RandomKeySameValueAppend ||
op == StreamOperation.SameKeySameValueAppend))
fs_bw.Write(tmp[i]);
else
*/
fs_bw.Write(singleval);
fout.Flush(true);
fs_bw.Close();
}
logger.Log("End Stream Append");
}
}
else
{
// Populate the keys and the values
Random random = new Random();
Byte[] val = new Byte[val_size];
if (ptype == StreamFactory.StreamPhysicalType.File)
{
long[] offsets = new long[num_operations];
for (int i = 0; i < num_operations; ++i)
{
offsets[i] = random.Next(0, num_operations - 1) * val_size;
}
// ValueDataStream type
logger.Log("Start Stream Open");
FileStream fin = new FileStream(exp_dir + "/DiskRaw", FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
BinaryReader fs_br = new BinaryReader(fin);
logger.Log("End Stream Open");
logger.Log("Start Stream Get");
logger.Log("Start Stream GetAll");
for (int i = 0; i < num_operations; ++i)
{
fs_br.BaseStream.Seek(offsets[i], SeekOrigin.Begin);
fs_br.Read(val, 0, val_size);
}
fs_br.Close();
logger.Log("End Stream Get");
logger.Log("End Stream GetAll");
}
else
{
string[] filenames = new string[num_operations];
for (int i = 0; i < num_operations; ++i)
{
filenames[i] = exp_dir + "/DiskRaw" + random.Next(0, num_operations);
}
// FileDataStream type
logger.Log("Start Stream Get");
logger.Log("Start Stream GetAll");
for (int i = 0; i < num_operations; ++i)
{
FileStream fin = new FileStream(filenames[i], FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
BinaryReader fs_br = new BinaryReader(fin);
fs_br.Read(val, 0, val_size);
fs_br.Close();
}
logger.Log("End Stream Get");
logger.Log("End Stream GetAll");
}
}
CostsHelper costhelper = new CostsHelper();
logger.Log(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(exp_dir, dataRelated:true)/1000.0f);
logger.Log(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(exp_dir, dataRelated:false)/1000.0f);
return logger;
}
public List<string> Dump(string CallerName,
string HomeName, string AppName, string StreamName,
StreamType stream_type,
StreamOperation stream_op,
StreamFactory.StreamPhysicalType ptype,
Byte[] value, int num_operations,
SynchronizerType synctype,
bool doCosts= false)
{
List<string> exp_details = new List<string>();
exp_details.Add(exp_id);
File.AppendAllLines(exp_directory + "/exp", exp_details);
return exp_details;
}
public void Run(string CallerName,
string HomeName, string AppName, string StreamName,
string RandName,
long stime, long etime,
StreamType stream_type,
StreamOperation stream_op,
StreamFactory.StreamPhysicalType ptype,
CompressionType ctype, int ChunkSize , int ThreadPoolSize,
Byte[] value, int num_operations,
SynchronizerType synctype,
int max_key = 0,
string address = null,
bool doCosts= false,
bool doRaw = false)
{
// Set experiment directory
CallerInfo ci = new CallerInfo(null, CallerName, CallerName, 1);
exp_directory = Path.GetFullPath((null != ci.workingDir) ? ci.workingDir : Directory.GetCurrentDirectory());
exp_directory = exp_directory + "/" + HomeName + "/" + AppName + "/" + StreamName;
if (max_key == 0)
max_key = num_operations;
// Set a description/tag for the experiment
this.exp_id = "Directory: " + HomeName + "/" + AppName + "/" + StreamName +
" Caller:" + CallerName
+ " Stream Type:" + stream_type + " Stream Op: " + stream_op + " Stream Ptype: " + ptype + " Compression Type: " + ctype
+ " Value size: " + value.Length
+ " num_operations: " + max_key
+ " actual_num_ops: " + num_operations
+ " Sync type: " + synctype
+ " Do costs? " + doCosts + "Chunk Size: " + ChunkSize+ " ThreadPool Size:" +ThreadPoolSize;
this.compressed_exp_id =
" ST:" + stream_type + " OP: " + stream_op + " PT: " + ptype + " CT: " + ctype
+ " VS: " + value.Length
+ " I:" + num_operations
+ " MK:" + max_key
+ " SYNC: " + synctype+ " chsize: "+ChunkSize + " nThreads: "+ThreadPoolSize ;
// Set remote storage server account info
string AzureaccountName = ConfigurationManager.AppSettings.Get("AccountName");
string AzureaccountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey");
string S3accountName = ConfigurationManager.AppSettings.Get("S3AccountName");
string S3accountKey = ConfigurationManager.AppSettings.Get("S3AccountSharedKey");
LocationInfo Li;
if (synctype == SynchronizerType.Azure)
Li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
else if (synctype == SynchronizerType.AmazonS3)
Li = new LocationInfo(S3accountName, S3accountKey, SynchronizerType.AmazonS3);
else
Li = null;
StreamFactory sf = StreamFactory.Instance;
IStream stream = null;
FqStreamID streamid = new FqStreamID(HomeName, AppName, StreamName);
// Set op : R/W
StreamFactory.StreamOp rw;
if (stream_op == StreamOperation.RandomKeyRandomValueAppend
|| stream_op == StreamOperation.RandomKeySameValueAppend
|| stream_op == StreamOperation.SameKeyRandomValueAppend
|| stream_op == StreamOperation.SameKeySameValueAppend)
{
rw = StreamFactory.StreamOp.Write;
}
else
{
rw = StreamFactory.StreamOp.Read;
}
// Initialize costs
CostsHelper costhelper = null;
double baselineStorageKV = 0;
if (doCosts)
{
costhelper = new CostsHelper();
costhelper.getCurrentCpuUsage();
costhelper.getNetworkUsage();
}
if (stream_type == StreamType.CloudRaw)
{
if (!Directory.Exists(exp_directory))
{
Directory.CreateDirectory(exp_directory);
}
Logger logger = new Logger();
Byte[] val = new Byte[value.Length * num_operations];
// DateTime Date = new DateTime(DateTime.UtcNow.Ticks);
// string cname = String.Format("CloudRaw-{0}", Date.ToString("yyyy-MM-dd"));
// string bname = String.Format("{0}", Date.ToString("HH-mm-ss"));
// string cname = String.Format("cloudraw-{0}", RandomString(4));
// string bname = String.Format("{0}", RandomString(4));
string cname = String.Format("cloudraw-{0}", RandName);
string bname = String.Format("{0}", RandName);
if (stream_op == StreamOperation.RandomKeyGet ||
stream_op == StreamOperation.RandomKeyGetMultipleSegments ||
stream_op == StreamOperation.RandomKeyGetAll)
{
doRawCloudPerf(val, SynchronizerType.Azure,
SynchronizeDirection.Download, exp_directory, logger, containerName: cname, blobName: bname);
logger.Dump(exp_directory + "/log");
}
else
{
doRawCloudPerf(val, SynchronizerType.Azure,
SynchronizeDirection.Upload, exp_directory, logger, containerName: cname, blobName: bname);
logger.Dump(exp_directory + "/log");
}
return;
}
if (stream_type == StreamType.DiskRaw)
{
if (!Directory.Exists(exp_directory))
{
Directory.CreateDirectory(exp_directory);
}
Logger logger = doDiskRaw(stream_op, num_operations, value.Length, ptype, exp_directory);
logger.Dump(exp_directory + "/log");
return;
}
// Are we getting raw disk throughput?
if (stream_type == StreamType.Raw)
{
string ret = doDiskSpeed((value.Length * num_operations)/1000 + "K", value.Length/1000 + "K", rw);
if (!Directory.Exists(exp_directory))
{
Directory.CreateDirectory(exp_directory);
}
File.WriteAllText(exp_directory + "/log", ret);
return;
}
// Populate the keys and the values
Random random = new Random(DateTime.Now.Millisecond);
StrKey[] keys = new StrKey[max_key];
for (int i = 0; i < max_key; ++i)
{
keys[i] = new StrKey("" + i);
}
/*
List<ByteValue> vals = new List<ByteValue>(num_operations);
Byte[][] tmp = new Byte[num_operations][];
for (int i = 0; i < num_operations; ++i)
{
tmp[i] = new Byte[value.Length];
random.NextBytes(tmp[i]);
}
for (int i = 0; i < num_operations; ++i)
{
keys[i] = new StrKey("" + i);
vals.Add(new ByteValue(tmp[i]));
// vals[i] = new ByteValue(tmp);
}
*/
Logger log = new Logger();
// Open stream for different types of experiments
if (stream_type == StreamType.Local && ptype == StreamFactory.StreamPhysicalType.File)
{
stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamPhysicalType.File)
{
stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamPhysicalType.File)
{
stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamPhysicalType.File)
{
stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.Local && ptype == StreamFactory.StreamPhysicalType.Directory)
{
stream = sf.openDirStream<StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamPhysicalType.Directory)
{
stream = sf.openDirStream<StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamPhysicalType.Directory)
{
stream = sf.openDirStream<StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamPhysicalType.Directory)
{
stream = sf.openDirStream<StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else
{
return;
}
if (stream_op == StreamOperation.RandomKeyRandomValueAppend)
{
List<ByteValue> vals = new List<ByteValue>(num_operations);
Byte[][] tmp = new Byte[num_operations][];
for (int i = 0; i < num_operations; ++i)
{
tmp[i] = new Byte[value.Length];
random.NextBytes(tmp[i]);
}
for (int i = 0; i < num_operations; ++i)
{
vals.Add(new ByteValue(tmp[i]));
}
for (int i = 0; i < num_operations; ++i)
{
baselineStorageKV += keys[i].Size();
baselineStorageKV += vals[i].Size();
stream.Append(keys[i], vals[i]);
}
stream.Close();
}
else if (stream_op == StreamOperation.RandomKeySameValueAppend)
{
Byte[] singleval = new Byte[value.Length];
random.NextBytes(singleval);
ByteValue singlebv = new ByteValue(singleval);
for (int i = 0; i < num_operations; ++i)
{
baselineStorageKV += keys[i].Size();
baselineStorageKV += value.Length;
stream.Append(keys[i], singlebv);
}
stream.Close();
}
else if (stream_op == StreamOperation.SameKeySameValueAppend)
{
StrKey key = new StrKey("ExpKey");
Byte[] singleval = new Byte[value.Length];
random.NextBytes(singleval);
ByteValue singlebv = new ByteValue(singleval);
for (int i = 0; i < num_operations; ++i)
{
stream.Append(key, singlebv);
// System.Threading.Thread.Sleep(10);
}
stream.Close();
}
else if (stream_op == StreamOperation.RandomKeyGet || stream_op == StreamOperation.RandomKeyGetMultipleSegments)
{
for (int i = 0; i < num_operations; ++i)
{
stream.Get(keys[random.Next(0, max_key)]);
}
stream.Close();
}
else if (stream_op == StreamOperation.RandomKeyGetAll)
{
StrKey key = new StrKey("ExpKey");
for (int i = 0; i < num_operations; )
{
long st = 0;
long et = -1;
Console.WriteLine(stime + ":" + etime);
while (et < st)
{
st = RandomLong(stime, etime, random);
// et = RandomLong(stime, etime, random);
et = st + (10 * 10 * TimeSpan.TicksPerMillisecond);
}
Console.WriteLine(st + ":" + et);
IEnumerable<IDataItem> iterator = stream.GetAll(key, st, et);
foreach (IDataItem data in iterator)
{
data.GetVal();
++i;
if (i == num_operations)
break;
}
}
stream.Close();
}
else if (stream_op == StreamOperation.SameKeyRandomValueAppend)
{
StrKey key = new StrKey("ExpKey");
for (int i = 0; i < num_operations; ++i)
{
baselineStorageKV += key.Size();
// baselineStorageKV += vals[i].Size();
// stream.Append(key, vals[i]);
}
stream.Close();
}
else
{
for (int i = 0; i < num_operations; ++i)
{
stream.Get(new StrKey("" + random.Next(0,num_operations - 1)));
}
stream.Close();
}
// Dump the instrumentation logs
stream.DumpLogs(exp_directory + "/log");
// Collect costs usage
List<string> costs = new List<string>();
if (doCosts)
{
costs.Add(DateTime.UtcNow.Ticks + ": CPU: " + costhelper.getCurrentCpuUsage());
costs.Add(DateTime.UtcNow.Ticks + ": Network: " + costhelper.getNetworkUsage());
costs.Add(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated:true)/1000.0f);
costs.Add(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated:false)/1000.0f);
costs.Add(DateTime.UtcNow.Ticks + ": Baseline Storage: " + baselineStorageKV/1000.0f);
}
File.AppendAllLines(exp_directory + "/log", costs);
// sf.deleteStream(streamid, ci);
}
public ObservableStreamEvent(StreamOperation operation, IEnumerable<byte> data)
{
this.Operation = operation;
this.Data = data;
}
public StreamOperation m_op; // The operation to perform.
#endregion Fields
#region Constructors
/////////////////////////////////////////////////////////////////////////////////////////////
//
// CAsyncOperation class. - Private class used by CWavStream class.
//
/////////////////////////////////////////////////////////////////////////////////////////////
public CAsyncOperation(StreamOperation op)
{
TRACE(string.Format("CAsyncOperation::CAsyncOperation ({0})", op.ToString()));
m_op = op;
}
