public SyncingContext(SynchronizerType synchronizerType)
{
_logger = _logManager.GetClassLogger();
ISyncConfig syncConfig = new SyncConfig();
syncConfig.FastSync = synchronizerType == SynchronizerType.Fast;
ISnapshotableDb stateDb = new StateDb();
ISnapshotableDb codeDb = new StateDb();
BlockTree = new BlockTree(new MemDb(), new MemDb(), new MemDb(), new SingleReleaseSpecProvider(Constantinople.Instance, 1), NullTxPool.Instance, _logManager);
var stats = new NodeStatsManager(new StatsConfig(), _logManager);
SyncPeerPool = new EthSyncPeerPool(BlockTree, stats, syncConfig, _logManager);
NodeDataFeed feed = new NodeDataFeed(codeDb, stateDb, _logManager);
NodeDataDownloader nodeDataDownloader = new NodeDataDownloader(SyncPeerPool, feed, _logManager);
Synchronizer = new Synchronizer(BlockTree,
TestBlockValidator.AlwaysValid,
TestSealValidator.AlwaysValid,
SyncPeerPool, syncConfig, nodeDataDownloader, _logManager);
SyncServer = new SyncServer(stateDb, codeDb, BlockTree, NullReceiptStorage.Instance, TestSealValidator.AlwaysValid, SyncPeerPool, Synchronizer, _logManager);
SyncPeerPool.Start();
Synchronizer.Start();
Synchronizer.SyncEvent += SynchronizerOnSyncEvent;
AllInstances.Add(this);
}
public SyncingContext(SynchronizerType synchronizerType)
{
_logger = _logManager.GetClassLogger();
ISyncConfig syncConfig = synchronizerType == SynchronizerType.Fast ? SyncConfig.WithFastSync : SyncConfig.WithFullSyncOnly;
IDbProvider dbProvider = TestMemDbProvider.Init();
IDb stateDb = new MemDb();
IDb codeDb = dbProvider.CodeDb;
MemDb blockInfoDb = new MemDb();
BlockTree = new BlockTree(new MemDb(), new MemDb(), blockInfoDb, new ChainLevelInfoRepository(blockInfoDb), new SingleReleaseSpecProvider(Constantinople.Instance, 1), NullBloomStorage.Instance, _logManager);
ITimerFactory timerFactory = Substitute.For <ITimerFactory>();
NodeStatsManager stats = new NodeStatsManager(timerFactory, _logManager);
SyncPeerPool = new SyncPeerPool(BlockTree, stats, 25, _logManager);
SyncProgressResolver syncProgressResolver = new SyncProgressResolver(
BlockTree,
NullReceiptStorage.Instance,
stateDb,
new MemDb(),
new TrieStore(stateDb, LimboLogs.Instance),
syncConfig,
_logManager);
MultiSyncModeSelector syncModeSelector = new MultiSyncModeSelector(syncProgressResolver, SyncPeerPool, syncConfig, _logManager);
Synchronizer = new Synchronizer(
dbProvider,
MainnetSpecProvider.Instance,
BlockTree,
NullReceiptStorage.Instance,
Always.Valid,
Always.Valid,
SyncPeerPool,
stats,
syncModeSelector,
syncConfig,
_logManager);
SyncServer = new SyncServer(
stateDb,
codeDb,
BlockTree,
NullReceiptStorage.Instance,
Always.Valid,
Always.Valid,
SyncPeerPool,
syncModeSelector,
syncConfig,
new WitnessCollector(new MemDb(), LimboLogs.Instance),
_logManager);
SyncPeerPool.Start();
Synchronizer.Start();
Synchronizer.SyncEvent += SynchronizerOnSyncEvent;
AllInstances.Add(this);
}
public List <string> Dump(string CallerName,
string HomeName, string AppName, string StreamName,
StreamType stream_type,
StreamOperation stream_op,
StreamFactory.StreamDataType ptype,
Byte[] value, int num_operations,
SynchronizerType synctype,
bool doCosts = false)
{
List <string> exp_details = new List <string>();
exp_details.Add(exp_id);
File.AppendAllLines(exp_directory + "/exp", exp_details);
return(exp_details);
}
public SyncThreadsTests(SynchronizerType synchronizerType)
{
_synchronizerType = synchronizerType;
}
public SynchronizerTests(SynchronizerType synchronizerType)
{
_synchronizerType = synchronizerType;
}
public WhenImplementation(SynchronizerType synchronizerType)
{
_synchronizerType = synchronizerType;
}
public void Run(string CallerName,
string HomeName, string AppName, string StreamName,
string RandName,
long stime, long etime,
StreamType stream_type,
StreamOperation stream_op,
StreamFactory.StreamPhysicalType ptype,
CompressionType ctype, int ChunkSize , int ThreadPoolSize,
Byte[] value, int num_operations,
SynchronizerType synctype,
int max_key = 0,
string address = null,
bool doCosts= false,
bool doRaw = false)
{
// Set experiment directory
CallerInfo ci = new CallerInfo(null, CallerName, CallerName, 1);
exp_directory = Path.GetFullPath((null != ci.workingDir) ? ci.workingDir : Directory.GetCurrentDirectory());
exp_directory = exp_directory + "/" + HomeName + "/" + AppName + "/" + StreamName;
if (max_key == 0)
max_key = num_operations;
// Set a description/tag for the experiment
this.exp_id = "Directory: " + HomeName + "/" + AppName + "/" + StreamName +
" Caller:" + CallerName
+ " Stream Type:" + stream_type + " Stream Op: " + stream_op + " Stream Ptype: " + ptype + " Compression Type: " + ctype
+ " Value size: " + value.Length
+ " num_operations: " + max_key
+ " actual_num_ops: " + num_operations
+ " Sync type: " + synctype
+ " Do costs? " + doCosts + "Chunk Size: " + ChunkSize+ " ThreadPool Size:" +ThreadPoolSize;
this.compressed_exp_id =
" ST:" + stream_type + " OP: " + stream_op + " PT: " + ptype + " CT: " + ctype
+ " VS: " + value.Length
+ " I:" + num_operations
+ " MK:" + max_key
+ " SYNC: " + synctype+ " chsize: "+ChunkSize + " nThreads: "+ThreadPoolSize ;
// Set remote storage server account info
string AzureaccountName = ConfigurationManager.AppSettings.Get("AccountName");
string AzureaccountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey");
string S3accountName = ConfigurationManager.AppSettings.Get("S3AccountName");
string S3accountKey = ConfigurationManager.AppSettings.Get("S3AccountSharedKey");
LocationInfo Li;
if (synctype == SynchronizerType.Azure)
Li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
else if (synctype == SynchronizerType.AmazonS3)
Li = new LocationInfo(S3accountName, S3accountKey, SynchronizerType.AmazonS3);
else
Li = null;
StreamFactory sf = StreamFactory.Instance;
IStream stream = null;
FqStreamID streamid = new FqStreamID(HomeName, AppName, StreamName);
// Set op : R/W
StreamFactory.StreamOp rw;
if (stream_op == StreamOperation.RandomKeyRandomValueAppend
|| stream_op == StreamOperation.RandomKeySameValueAppend
|| stream_op == StreamOperation.SameKeyRandomValueAppend
|| stream_op == StreamOperation.SameKeySameValueAppend)
{
rw = StreamFactory.StreamOp.Write;
}
else
{
rw = StreamFactory.StreamOp.Read;
}
// Initialize costs
CostsHelper costhelper = null;
double baselineStorageKV = 0;
if (doCosts)
{
costhelper = new CostsHelper();
costhelper.getCurrentCpuUsage();
costhelper.getNetworkUsage();
}
if (stream_type == StreamType.CloudRaw)
{
if (!Directory.Exists(exp_directory))
{
Directory.CreateDirectory(exp_directory);
}
Logger logger = new Logger();
Byte[] val = new Byte[value.Length * num_operations];
// DateTime Date = new DateTime(DateTime.UtcNow.Ticks);
// string cname = String.Format("CloudRaw-{0}", Date.ToString("yyyy-MM-dd"));
// string bname = String.Format("{0}", Date.ToString("HH-mm-ss"));
// string cname = String.Format("cloudraw-{0}", RandomString(4));
// string bname = String.Format("{0}", RandomString(4));
string cname = String.Format("cloudraw-{0}", RandName);
string bname = String.Format("{0}", RandName);
if (stream_op == StreamOperation.RandomKeyGet ||
stream_op == StreamOperation.RandomKeyGetMultipleSegments ||
stream_op == StreamOperation.RandomKeyGetAll)
{
doRawCloudPerf(val, SynchronizerType.Azure,
SynchronizeDirection.Download, exp_directory, logger, containerName: cname, blobName: bname);
logger.Dump(exp_directory + "/log");
}
else
{
doRawCloudPerf(val, SynchronizerType.Azure,
SynchronizeDirection.Upload, exp_directory, logger, containerName: cname, blobName: bname);
logger.Dump(exp_directory + "/log");
}
return;
}
if (stream_type == StreamType.DiskRaw)
{
if (!Directory.Exists(exp_directory))
{
Directory.CreateDirectory(exp_directory);
}
Logger logger = doDiskRaw(stream_op, num_operations, value.Length, ptype, exp_directory);
logger.Dump(exp_directory + "/log");
return;
}
// Are we getting raw disk throughput?
if (stream_type == StreamType.Raw)
{
string ret = doDiskSpeed((value.Length * num_operations)/1000 + "K", value.Length/1000 + "K", rw);
if (!Directory.Exists(exp_directory))
{
Directory.CreateDirectory(exp_directory);
}
File.WriteAllText(exp_directory + "/log", ret);
return;
}
// Populate the keys and the values
Random random = new Random(DateTime.Now.Millisecond);
StrKey[] keys = new StrKey[max_key];
for (int i = 0; i < max_key; ++i)
{
keys[i] = new StrKey("" + i);
}
/*
List<ByteValue> vals = new List<ByteValue>(num_operations);
Byte[][] tmp = new Byte[num_operations][];
for (int i = 0; i < num_operations; ++i)
{
tmp[i] = new Byte[value.Length];
random.NextBytes(tmp[i]);
}
for (int i = 0; i < num_operations; ++i)
{
keys[i] = new StrKey("" + i);
vals.Add(new ByteValue(tmp[i]));
// vals[i] = new ByteValue(tmp);
}
*/
Logger log = new Logger();
// Open stream for different types of experiments
if (stream_type == StreamType.Local && ptype == StreamFactory.StreamPhysicalType.File)
{
stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamPhysicalType.File)
{
stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamPhysicalType.File)
{
stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamPhysicalType.File)
{
stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.Local && ptype == StreamFactory.StreamPhysicalType.Directory)
{
stream = sf.openDirStream<StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamPhysicalType.Directory)
{
stream = sf.openDirStream<StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamPhysicalType.Directory)
{
stream = sf.openDirStream<StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamPhysicalType.Directory)
{
stream = sf.openDirStream<StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else
{
return;
}
if (stream_op == StreamOperation.RandomKeyRandomValueAppend)
{
List<ByteValue> vals = new List<ByteValue>(num_operations);
Byte[][] tmp = new Byte[num_operations][];
for (int i = 0; i < num_operations; ++i)
{
tmp[i] = new Byte[value.Length];
random.NextBytes(tmp[i]);
}
for (int i = 0; i < num_operations; ++i)
{
vals.Add(new ByteValue(tmp[i]));
}
for (int i = 0; i < num_operations; ++i)
{
baselineStorageKV += keys[i].Size();
baselineStorageKV += vals[i].Size();
stream.Append(keys[i], vals[i]);
}
stream.Close();
}
else if (stream_op == StreamOperation.RandomKeySameValueAppend)
{
Byte[] singleval = new Byte[value.Length];
random.NextBytes(singleval);
ByteValue singlebv = new ByteValue(singleval);
for (int i = 0; i < num_operations; ++i)
{
baselineStorageKV += keys[i].Size();
baselineStorageKV += value.Length;
stream.Append(keys[i], singlebv);
}
stream.Close();
}
else if (stream_op == StreamOperation.SameKeySameValueAppend)
{
StrKey key = new StrKey("ExpKey");
Byte[] singleval = new Byte[value.Length];
random.NextBytes(singleval);
ByteValue singlebv = new ByteValue(singleval);
for (int i = 0; i < num_operations; ++i)
{
stream.Append(key, singlebv);
// System.Threading.Thread.Sleep(10);
}
stream.Close();
}
else if (stream_op == StreamOperation.RandomKeyGet || stream_op == StreamOperation.RandomKeyGetMultipleSegments)
{
for (int i = 0; i < num_operations; ++i)
{
stream.Get(keys[random.Next(0, max_key)]);
}
stream.Close();
}
else if (stream_op == StreamOperation.RandomKeyGetAll)
{
StrKey key = new StrKey("ExpKey");
for (int i = 0; i < num_operations; )
{
long st = 0;
long et = -1;
Console.WriteLine(stime + ":" + etime);
while (et < st)
{
st = RandomLong(stime, etime, random);
// et = RandomLong(stime, etime, random);
et = st + (10 * 10 * TimeSpan.TicksPerMillisecond);
}
Console.WriteLine(st + ":" + et);
IEnumerable<IDataItem> iterator = stream.GetAll(key, st, et);
foreach (IDataItem data in iterator)
{
data.GetVal();
++i;
if (i == num_operations)
break;
}
}
stream.Close();
}
else if (stream_op == StreamOperation.SameKeyRandomValueAppend)
{
StrKey key = new StrKey("ExpKey");
for (int i = 0; i < num_operations; ++i)
{
baselineStorageKV += key.Size();
// baselineStorageKV += vals[i].Size();
// stream.Append(key, vals[i]);
}
stream.Close();
}
else
{
for (int i = 0; i < num_operations; ++i)
{
stream.Get(new StrKey("" + random.Next(0,num_operations - 1)));
}
stream.Close();
}
// Dump the instrumentation logs
stream.DumpLogs(exp_directory + "/log");
// Collect costs usage
List<string> costs = new List<string>();
if (doCosts)
{
costs.Add(DateTime.UtcNow.Ticks + ": CPU: " + costhelper.getCurrentCpuUsage());
costs.Add(DateTime.UtcNow.Ticks + ": Network: " + costhelper.getNetworkUsage());
costs.Add(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated:true)/1000.0f);
costs.Add(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated:false)/1000.0f);
costs.Add(DateTime.UtcNow.Ticks + ": Baseline Storage: " + baselineStorageKV/1000.0f);
}
File.AppendAllLines(exp_directory + "/log", costs);
// sf.deleteStream(streamid, ci);
}
public List<string> Dump(string CallerName,
string HomeName, string AppName, string StreamName,
StreamType stream_type,
StreamOperation stream_op,
StreamFactory.StreamPhysicalType ptype,
Byte[] value, int num_operations,
SynchronizerType synctype,
bool doCosts= false)
{
List<string> exp_details = new List<string>();
exp_details.Add(exp_id);
File.AppendAllLines(exp_directory + "/exp", exp_details);
return exp_details;
}
/*
Sample call for upload:-
byte[] array = new byte[1024*1024*1024];
Random random = new Random();
random.NextBytes(array);
double timeTaken_Upload = Experiment.doRawCloudPerf(array, SynchronizerType.Azure, SynchronizeDirection.Upload, "fooContainer", "fooBlob");
double timeTaken_Download = Experiment.doRawCloudPerf(array, SynchronizerType.Azure, SynchronizeDirection.Download, "fooContainer", "fooBlob");
*
*
*/
public static double doRawCloudPerf(byte[] input, SynchronizerType synchronizerType,
SynchronizeDirection syncDirection, string exp_directory, Logger logger, string containerName=null, string blobName=null)
{
string accountName = ConfigurationManager.AppSettings.Get("AccountName");
string accountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey");
DateTime begin=DateTime.Now, end=DateTime.Now;
if (synchronizerType == SynchronizerType.Azure)
{
#region azure download/upload
if (containerName==null)
containerName = "testingraw";
if(blobName==null)
blobName = Guid.NewGuid().ToString();
CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentialsAccountAndKey(accountName, accountKey), true);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
if (syncDirection == SynchronizeDirection.Upload)
{
logger.Log("Start Stream Append");
container.CreateIfNotExist();
begin = DateTime.UtcNow;//////////////////////////////////////
try
{
using (MemoryStream memoryStream = new System.IO.MemoryStream(input))
{
CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName);
blockBlob.UploadFromStream(memoryStream);
}
}
catch (Exception e)
{
}
end = DateTime.UtcNow;//////////////////////////////////////
logger.Log("End Stream Append");
}
if (syncDirection == SynchronizeDirection.Download)
{
logger.Log("Start Stream Get");
logger.Log("Start Stream GetAll");
try
{
CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName);
byte[] blobContents = blockBlob.DownloadByteArray();
//if (File.Exists(blobName))
// File.Delete(blobName);
begin = DateTime.UtcNow;//////////////////////////////////////
// using (FileStream fs = new FileStream(blobName, FileMode.OpenOrCreate))
// {
byte[] contents = blockBlob.DownloadByteArray();
// fs.Write(contents, 0, contents.Length);
// }
}
catch (Exception e)
{
}
end = DateTime.UtcNow;//////////////////////////////////////
logger.Log("End Stream Get");
logger.Log("End Stream GetAll");
}
#endregion
}
else if (synchronizerType == SynchronizerType.AmazonS3)
{
#region amazon s3 stuff
if (containerName == null)
containerName = "testingraw";
if (blobName == null)
blobName = Guid.NewGuid().ToString();
AmazonS3Client amazonS3Client = new AmazonS3Client(accountName, accountKey);
if (syncDirection == SynchronizeDirection.Upload)
{
ListBucketsResponse response = amazonS3Client.ListBuckets();
foreach (S3Bucket bucket in response.Buckets)
{
if (bucket.BucketName == containerName)
{
break;
}
}
amazonS3Client.PutBucket(new PutBucketRequest().WithBucketName(containerName));
begin = DateTime.UtcNow;//////////////////////////////////////
MemoryStream ms = new MemoryStream();
ms.Write(input, 0, input.Length);
PutObjectRequest request = new PutObjectRequest();
request.WithBucketName(containerName);
request.WithKey(blobName);
request.InputStream = ms;
amazonS3Client.PutObject(request);
end = DateTime.UtcNow;//////////////////////////////////////
}
if (syncDirection == SynchronizeDirection.Download)
{
if (File.Exists(blobName))
File.Delete(blobName);
begin = DateTime.UtcNow;//////////////////////////////////////
GetObjectRequest request = new GetObjectRequest();
request.WithBucketName(containerName);
request.WithKey(blobName);
GetObjectResponse response = amazonS3Client.GetObject(request);
var localFileStream = File.Create(blobName);
response.ResponseStream.CopyTo(localFileStream);
localFileStream.Close();
end = DateTime.UtcNow;//////////////////////////////////////
}
#endregion
}
else
{
throw new InvalidDataException("syncronizer type is not valid");
}
return (end - begin).TotalMilliseconds;// return total time to upload in milliseconds
}
public LocationInfo(string accName, string accKey, SynchronizerType St)
{
accountName = accName;
accountKey = accKey;
st = St;
}
void Start()
{
if (type == SynchronizerType.ServerClientAuto) {
type = Application.platform == RuntimePlatform.Android ? SynchronizerType.Client : SynchronizerType.Server;
}
if (type == SynchronizerType.Client) {
if (File.Exists(HEADSET_PATH)) {
using (var headsetStream = File.OpenRead(HEADSET_PATH)) {
var reader = new BinaryReader(headsetStream);
var num = reader.ReadInt32();
SetHeadsetNumber(num, false);
}
}
clientSock = new Socket(AddressFamily.InterNetwork,
SocketType.Stream, ProtocolType.Tcp);
var ipEnd = new IPEndPoint(IPAddress.Any, BACKFILL_PORT);
clientSock.Bind(ipEnd);
clientSock.Listen(100);
}
if(type != SynchronizerType.Client &&
type != SynchronizerType.Playback) {
// Start receiving real packets
masterStream.StartReceiving();
}
if (type == SynchronizerType.Recorder) {
if (Directory.Exists(recordPath)) {
var lastFrame = LastRecordedFrameNumber();
LoadRecordedFrame(lastFrame);
} else {
Directory.CreateDirectory(recordPath);
}
using (var s = File.OpenWrite(LastFrameNumberPath)) {
var b = new BinaryWriter(s);
b.Write(0u);
}
}
}
public LocationInfo(string accName, string accKey, SynchronizerType St)
{
accountName = accName;
accountKey = accKey;
st = St;
}
public void Run(string CallerName,
string HomeName, string AppName, string StreamName,
string RandName,
long stime, long etime,
StreamType stream_type,
StreamOperation stream_op,
StreamFactory.StreamDataType ptype,
CompressionType ctype, int ChunkSize, int ThreadPoolSize,
Byte[] value, int num_operations,
SynchronizerType synctype,
int max_key = 0,
string address = null,
bool doCosts = false,
bool doRaw = false)
{
// Set experiment directory
CallerInfo ci = new CallerInfo(null, CallerName, CallerName, 1);
exp_directory = Path.GetFullPath((null != ci.workingDir) ? ci.workingDir : Directory.GetCurrentDirectory());
exp_directory = exp_directory + "/" + HomeName + "/" + AppName + "/" + StreamName;
if (max_key == 0)
{
max_key = num_operations;
}
// Set a description/tag for the experiment
this.exp_id = "Directory: " + HomeName + "/" + AppName + "/" + StreamName +
" Caller:" + CallerName
+ " Stream Type:" + stream_type + " Stream Op: " + stream_op + " Stream Ptype: " + ptype + " Compression Type: " + ctype
+ " Value size: " + value.Length
+ " num_operations: " + max_key
+ " actual_num_ops: " + num_operations
+ " Sync type: " + synctype
+ " Do costs? " + doCosts + "Chunk Size: " + ChunkSize + " ThreadPool Size:" + ThreadPoolSize;
this.compressed_exp_id =
" ST:" + stream_type + " OP: " + stream_op + " PT: " + ptype + " CT: " + ctype
+ " VS: " + value.Length
+ " I:" + num_operations
+ " MK:" + max_key
+ " SYNC: " + synctype + " chsize: " + ChunkSize + " nThreads: " + ThreadPoolSize;
// Set remote storage server account info
string AzureaccountName = ConfigurationManager.AppSettings.Get("AccountName");
string AzureaccountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey");
string S3accountName = ConfigurationManager.AppSettings.Get("S3AccountName");
string S3accountKey = ConfigurationManager.AppSettings.Get("S3AccountSharedKey");
LocationInfo Li;
if (synctype == SynchronizerType.Azure)
{
Li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
}
else if (synctype == SynchronizerType.AmazonS3)
{
Li = new LocationInfo(S3accountName, S3accountKey, SynchronizerType.AmazonS3);
}
else
{
Li = null;
}
StreamFactory sf = StreamFactory.Instance;
IStream stream = null;
FqStreamID streamid = new FqStreamID(HomeName, AppName, StreamName);
// Set op : R/W
StreamFactory.StreamOp rw;
if (stream_op == StreamOperation.RandomKeyRandomValueAppend ||
stream_op == StreamOperation.RandomKeySameValueAppend ||
stream_op == StreamOperation.SameKeyRandomValueAppend ||
stream_op == StreamOperation.SameKeySameValueAppend)
{
rw = StreamFactory.StreamOp.Write;
}
else
{
rw = StreamFactory.StreamOp.Read;
}
// Initialize costs
CostsHelper costhelper = null;
double baselineStorageKV = 0;
if (doCosts)
{
costhelper = new CostsHelper();
costhelper.getCurrentCpuUsage();
costhelper.getNetworkUsage();
}
if (stream_type == StreamType.CloudRaw)
{
if (!Directory.Exists(exp_directory))
{
Directory.CreateDirectory(exp_directory);
}
Logger logger = new Logger();
Byte[] val = new Byte[value.Length * num_operations];
// DateTime Date = new DateTime(DateTime.UtcNow.Ticks);
// string cname = String.Format("CloudRaw-{0}", Date.ToString("yyyy-MM-dd"));
// string bname = String.Format("{0}", Date.ToString("HH-mm-ss"));
// string cname = String.Format("cloudraw-{0}", RandomString(4));
// string bname = String.Format("{0}", RandomString(4));
string cname = String.Format("cloudraw-{0}", RandName);
string bname = String.Format("{0}", RandName);
if (stream_op == StreamOperation.RandomKeyGet ||
stream_op == StreamOperation.RandomKeyGetMultipleSegments ||
stream_op == StreamOperation.RandomKeyGetAll)
{
doRawCloudPerf(val, SynchronizerType.Azure,
SynchronizeDirection.Download, exp_directory, logger, containerName: cname, blobName: bname);
logger.Dump(exp_directory + "/log");
}
else
{
doRawCloudPerf(val, SynchronizerType.Azure,
SynchronizeDirection.Upload, exp_directory, logger, containerName: cname, blobName: bname);
logger.Dump(exp_directory + "/log");
}
return;
}
if (stream_type == StreamType.DiskRaw)
{
if (!Directory.Exists(exp_directory))
{
Directory.CreateDirectory(exp_directory);
}
Logger logger = doDiskRaw(stream_op, num_operations, value.Length, ptype, exp_directory);
logger.Dump(exp_directory + "/log");
return;
}
// Are we getting raw disk throughput?
if (stream_type == StreamType.Raw)
{
string ret = doDiskSpeed((value.Length * num_operations) / 1000 + "K", value.Length / 1000 + "K", rw);
if (!Directory.Exists(exp_directory))
{
Directory.CreateDirectory(exp_directory);
}
File.WriteAllText(exp_directory + "/log", ret);
return;
}
// Populate the keys and the values
Random random = new Random(DateTime.Now.Millisecond);
StrKey[] keys = new StrKey[max_key];
for (int i = 0; i < max_key; ++i)
{
keys[i] = new StrKey("" + i);
}
/*
* List<ByteValue> vals = new List<ByteValue>(num_operations);
* Byte[][] tmp = new Byte[num_operations][];
* for (int i = 0; i < num_operations; ++i)
* {
* tmp[i] = new Byte[value.Length];
* random.NextBytes(tmp[i]);
* }
*
* for (int i = 0; i < num_operations; ++i)
* {
* keys[i] = new StrKey("" + i);
* vals.Add(new ByteValue(tmp[i]));
* // vals[i] = new ByteValue(tmp);
* }
*/
Logger log = new Logger();
// Open stream for different types of experiments
if (stream_type == StreamType.Local && ptype == StreamFactory.StreamDataType.Values)
{
stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamDataType.Values)
{
stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamDataType.Values)
{
stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamDataType.Values)
{
stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.Local && ptype == StreamFactory.StreamDataType.Files)
{
stream = sf.openFileDataStream <StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamDataType.Files)
{
stream = sf.openFileDataStream <StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamDataType.Files)
{
stream = sf.openFileDataStream <StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamDataType.Files)
{
stream = sf.openFileDataStream <StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
}
else
{
return;
}
if (stream_op == StreamOperation.RandomKeyRandomValueAppend)
{
List <ByteValue> vals = new List <ByteValue>(num_operations);
Byte[][] tmp = new Byte[num_operations][];
for (int i = 0; i < num_operations; ++i)
{
tmp[i] = new Byte[value.Length];
random.NextBytes(tmp[i]);
}
for (int i = 0; i < num_operations; ++i)
{
vals.Add(new ByteValue(tmp[i]));
}
for (int i = 0; i < num_operations; ++i)
{
baselineStorageKV += keys[i].Size();
baselineStorageKV += vals[i].Size();
stream.Append(keys[i], vals[i]);
}
stream.Close();
}
else if (stream_op == StreamOperation.RandomKeySameValueAppend)
{
Byte[] singleval = new Byte[value.Length];
random.NextBytes(singleval);
ByteValue singlebv = new ByteValue(singleval);
for (int i = 0; i < num_operations; ++i)
{
baselineStorageKV += keys[i].Size();
baselineStorageKV += value.Length;
stream.Append(keys[i], singlebv);
}
stream.Close();
}
else if (stream_op == StreamOperation.SameKeySameValueAppend)
{
StrKey key = new StrKey("ExpKey");
Byte[] singleval = new Byte[value.Length];
random.NextBytes(singleval);
ByteValue singlebv = new ByteValue(singleval);
for (int i = 0; i < num_operations; ++i)
{
stream.Append(key, singlebv);
// System.Threading.Thread.Sleep(10);
}
stream.Close();
}
else if (stream_op == StreamOperation.RandomKeyGet || stream_op == StreamOperation.RandomKeyGetMultipleSegments)
{
for (int i = 0; i < num_operations; ++i)
{
stream.Get(keys[random.Next(0, max_key)]);
}
stream.Close();
}
else if (stream_op == StreamOperation.RandomKeyGetAll)
{
StrKey key = new StrKey("ExpKey");
for (int i = 0; i < num_operations;)
{
long st = 0;
long et = -1;
Console.WriteLine(stime + ":" + etime);
while (et < st)
{
st = RandomLong(stime, etime, random);
// et = RandomLong(stime, etime, random);
et = st + (10 * 10 * TimeSpan.TicksPerMillisecond);
}
Console.WriteLine(st + ":" + et);
IEnumerable <IDataItem> iterator = stream.GetAll(key, st, et);
foreach (IDataItem data in iterator)
{
data.GetVal();
++i;
if (i == num_operations)
{
break;
}
}
}
stream.Close();
}
else if (stream_op == StreamOperation.SameKeyRandomValueAppend)
{
StrKey key = new StrKey("ExpKey");
for (int i = 0; i < num_operations; ++i)
{
baselineStorageKV += key.Size();
// baselineStorageKV += vals[i].Size();
// stream.Append(key, vals[i]);
}
stream.Close();
}
else
{
for (int i = 0; i < num_operations; ++i)
{
stream.Get(new StrKey("" + random.Next(0, num_operations - 1)));
}
stream.Close();
}
// Dump the instrumentation logs
stream.DumpLogs(exp_directory + "/log");
// Collect costs usage
List <string> costs = new List <string>();
if (doCosts)
{
costs.Add(DateTime.UtcNow.Ticks + ": CPU: " + costhelper.getCurrentCpuUsage());
costs.Add(DateTime.UtcNow.Ticks + ": Network: " + costhelper.getNetworkUsage());
costs.Add(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated: true) / 1000.0f);
costs.Add(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated: false) / 1000.0f);
costs.Add(DateTime.UtcNow.Ticks + ": Baseline Storage: " + baselineStorageKV / 1000.0f);
}
File.AppendAllLines(exp_directory + "/log", costs);
// sf.deleteStream(streamid, ci);
}
/*
* Sample call for upload:-
* byte[] array = new byte[1024*1024*1024];
* Random random = new Random();
* random.NextBytes(array);
* double timeTaken_Upload = Experiment.doRawCloudPerf(array, SynchronizerType.Azure, SynchronizeDirection.Upload, "fooContainer", "fooBlob");
* double timeTaken_Download = Experiment.doRawCloudPerf(array, SynchronizerType.Azure, SynchronizeDirection.Download, "fooContainer", "fooBlob");
*
*
*/
public static double doRawCloudPerf(byte[] input, SynchronizerType synchronizerType,
SynchronizeDirection syncDirection, string exp_directory, Logger logger, string containerName = null, string blobName = null)
{
string accountName = ConfigurationManager.AppSettings.Get("AccountName");
string accountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey");
DateTime begin = DateTime.Now, end = DateTime.Now;
if (synchronizerType == SynchronizerType.Azure)
{
#region azure download/upload
if (containerName == null)
{
containerName = "testingraw";
}
if (blobName == null)
{
blobName = Guid.NewGuid().ToString();
}
CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentialsAccountAndKey(accountName, accountKey), true);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
if (syncDirection == SynchronizeDirection.Upload)
{
logger.Log("Start Stream Append");
container.CreateIfNotExist();
begin = DateTime.UtcNow;//////////////////////////////////////
try
{
using (MemoryStream memoryStream = new System.IO.MemoryStream(input))
{
CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName);
blockBlob.UploadFromStream(memoryStream);
}
}
catch (Exception e)
{
Console.WriteLine("exception: " + e);
}
end = DateTime.UtcNow;//////////////////////////////////////
logger.Log("End Stream Append");
}
if (syncDirection == SynchronizeDirection.Download)
{
logger.Log("Start Stream Get");
logger.Log("Start Stream GetAll");
try
{
CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName);
byte[] blobContents = blockBlob.DownloadByteArray();
//if (File.Exists(blobName))
// File.Delete(blobName);
begin = DateTime.UtcNow;//////////////////////////////////////
// using (FileStream fs = new FileStream(blobName, FileMode.OpenOrCreate))
// {
byte[] contents = blockBlob.DownloadByteArray();
// fs.Write(contents, 0, contents.Length);
// }
}
catch (Exception e)
{
Console.WriteLine("exception: " + e);
}
end = DateTime.UtcNow;//////////////////////////////////////
logger.Log("End Stream Get");
logger.Log("End Stream GetAll");
}
#endregion
}
else if (synchronizerType == SynchronizerType.AmazonS3)
{
#region amazon s3 stuff
if (containerName == null)
{
containerName = "testingraw";
}
if (blobName == null)
{
blobName = Guid.NewGuid().ToString();
}
AmazonS3Client amazonS3Client = new AmazonS3Client(accountName, accountKey);
if (syncDirection == SynchronizeDirection.Upload)
{
ListBucketsResponse response = amazonS3Client.ListBuckets();
foreach (S3Bucket bucket in response.Buckets)
{
if (bucket.BucketName == containerName)
{
break;
}
}
amazonS3Client.PutBucket(new PutBucketRequest().WithBucketName(containerName));
begin = DateTime.UtcNow;//////////////////////////////////////
MemoryStream ms = new MemoryStream();
ms.Write(input, 0, input.Length);
PutObjectRequest request = new PutObjectRequest();
request.WithBucketName(containerName);
request.WithKey(blobName);
request.InputStream = ms;
amazonS3Client.PutObject(request);
end = DateTime.UtcNow;//////////////////////////////////////
}
if (syncDirection == SynchronizeDirection.Download)
{
if (File.Exists(blobName))
{
File.Delete(blobName);
}
begin = DateTime.UtcNow;//////////////////////////////////////
GetObjectRequest request = new GetObjectRequest();
request.WithBucketName(containerName);
request.WithKey(blobName);
GetObjectResponse response = amazonS3Client.GetObject(request);
var localFileStream = File.Create(blobName);
response.ResponseStream.CopyTo(localFileStream);
localFileStream.Close();
end = DateTime.UtcNow;//////////////////////////////////////
}
#endregion
}
else
{
throw new InvalidDataException("syncronizer type is not valid");
}
return((end - begin).TotalMilliseconds);// return total time to upload in milliseconds
}
