public SyncingContext(SynchronizerType synchronizerType) { _logger = _logManager.GetClassLogger(); ISyncConfig syncConfig = new SyncConfig(); syncConfig.FastSync = synchronizerType == SynchronizerType.Fast; ISnapshotableDb stateDb = new StateDb(); ISnapshotableDb codeDb = new StateDb(); BlockTree = new BlockTree(new MemDb(), new MemDb(), new MemDb(), new SingleReleaseSpecProvider(Constantinople.Instance, 1), NullTxPool.Instance, _logManager); var stats = new NodeStatsManager(new StatsConfig(), _logManager); SyncPeerPool = new EthSyncPeerPool(BlockTree, stats, syncConfig, _logManager); NodeDataFeed feed = new NodeDataFeed(codeDb, stateDb, _logManager); NodeDataDownloader nodeDataDownloader = new NodeDataDownloader(SyncPeerPool, feed, _logManager); Synchronizer = new Synchronizer(BlockTree, TestBlockValidator.AlwaysValid, TestSealValidator.AlwaysValid, SyncPeerPool, syncConfig, nodeDataDownloader, _logManager); SyncServer = new SyncServer(stateDb, codeDb, BlockTree, NullReceiptStorage.Instance, TestSealValidator.AlwaysValid, SyncPeerPool, Synchronizer, _logManager); SyncPeerPool.Start(); Synchronizer.Start(); Synchronizer.SyncEvent += SynchronizerOnSyncEvent; AllInstances.Add(this); }
public SyncingContext(SynchronizerType synchronizerType) { _logger = _logManager.GetClassLogger(); ISyncConfig syncConfig = synchronizerType == SynchronizerType.Fast ? SyncConfig.WithFastSync : SyncConfig.WithFullSyncOnly; IDbProvider dbProvider = TestMemDbProvider.Init(); IDb stateDb = new MemDb(); IDb codeDb = dbProvider.CodeDb; MemDb blockInfoDb = new MemDb(); BlockTree = new BlockTree(new MemDb(), new MemDb(), blockInfoDb, new ChainLevelInfoRepository(blockInfoDb), new SingleReleaseSpecProvider(Constantinople.Instance, 1), NullBloomStorage.Instance, _logManager); ITimerFactory timerFactory = Substitute.For <ITimerFactory>(); NodeStatsManager stats = new NodeStatsManager(timerFactory, _logManager); SyncPeerPool = new SyncPeerPool(BlockTree, stats, 25, _logManager); SyncProgressResolver syncProgressResolver = new SyncProgressResolver( BlockTree, NullReceiptStorage.Instance, stateDb, new MemDb(), new TrieStore(stateDb, LimboLogs.Instance), syncConfig, _logManager); MultiSyncModeSelector syncModeSelector = new MultiSyncModeSelector(syncProgressResolver, SyncPeerPool, syncConfig, _logManager); Synchronizer = new Synchronizer( dbProvider, MainnetSpecProvider.Instance, BlockTree, NullReceiptStorage.Instance, Always.Valid, Always.Valid, SyncPeerPool, stats, syncModeSelector, syncConfig, _logManager); SyncServer = new SyncServer( stateDb, codeDb, BlockTree, NullReceiptStorage.Instance, Always.Valid, Always.Valid, SyncPeerPool, syncModeSelector, syncConfig, new WitnessCollector(new MemDb(), LimboLogs.Instance), _logManager); SyncPeerPool.Start(); Synchronizer.Start(); Synchronizer.SyncEvent += SynchronizerOnSyncEvent; AllInstances.Add(this); }
public List <string> Dump(string CallerName, string HomeName, string AppName, string StreamName, StreamType stream_type, StreamOperation stream_op, StreamFactory.StreamDataType ptype, Byte[] value, int num_operations, SynchronizerType synctype, bool doCosts = false) { List <string> exp_details = new List <string>(); exp_details.Add(exp_id); File.AppendAllLines(exp_directory + "/exp", exp_details); return(exp_details); }
public SyncThreadsTests(SynchronizerType synchronizerType) { _synchronizerType = synchronizerType; }
public SynchronizerTests(SynchronizerType synchronizerType) { _synchronizerType = synchronizerType; }
public WhenImplementation(SynchronizerType synchronizerType) { _synchronizerType = synchronizerType; }
public void Run(string CallerName, string HomeName, string AppName, string StreamName, string RandName, long stime, long etime, StreamType stream_type, StreamOperation stream_op, StreamFactory.StreamPhysicalType ptype, CompressionType ctype, int ChunkSize , int ThreadPoolSize, Byte[] value, int num_operations, SynchronizerType synctype, int max_key = 0, string address = null, bool doCosts= false, bool doRaw = false) { // Set experiment directory CallerInfo ci = new CallerInfo(null, CallerName, CallerName, 1); exp_directory = Path.GetFullPath((null != ci.workingDir) ? ci.workingDir : Directory.GetCurrentDirectory()); exp_directory = exp_directory + "/" + HomeName + "/" + AppName + "/" + StreamName; if (max_key == 0) max_key = num_operations; // Set a description/tag for the experiment this.exp_id = "Directory: " + HomeName + "/" + AppName + "/" + StreamName + " Caller:" + CallerName + " Stream Type:" + stream_type + " Stream Op: " + stream_op + " Stream Ptype: " + ptype + " Compression Type: " + ctype + " Value size: " + value.Length + " num_operations: " + max_key + " actual_num_ops: " + num_operations + " Sync type: " + synctype + " Do costs? " + doCosts + "Chunk Size: " + ChunkSize+ " ThreadPool Size:" +ThreadPoolSize; this.compressed_exp_id = " ST:" + stream_type + " OP: " + stream_op + " PT: " + ptype + " CT: " + ctype + " VS: " + value.Length + " I:" + num_operations + " MK:" + max_key + " SYNC: " + synctype+ " chsize: "+ChunkSize + " nThreads: "+ThreadPoolSize ; // Set remote storage server account info string AzureaccountName = ConfigurationManager.AppSettings.Get("AccountName"); string AzureaccountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey"); string S3accountName = ConfigurationManager.AppSettings.Get("S3AccountName"); string S3accountKey = ConfigurationManager.AppSettings.Get("S3AccountSharedKey"); LocationInfo Li; if (synctype == SynchronizerType.Azure) Li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure); else if (synctype == SynchronizerType.AmazonS3) Li = new LocationInfo(S3accountName, S3accountKey, SynchronizerType.AmazonS3); else Li = null; StreamFactory sf = StreamFactory.Instance; IStream stream = null; FqStreamID streamid = new FqStreamID(HomeName, AppName, StreamName); // Set op : R/W StreamFactory.StreamOp rw; if (stream_op == StreamOperation.RandomKeyRandomValueAppend || stream_op == StreamOperation.RandomKeySameValueAppend || stream_op == StreamOperation.SameKeyRandomValueAppend || stream_op == StreamOperation.SameKeySameValueAppend) { rw = StreamFactory.StreamOp.Write; } else { rw = StreamFactory.StreamOp.Read; } // Initialize costs CostsHelper costhelper = null; double baselineStorageKV = 0; if (doCosts) { costhelper = new CostsHelper(); costhelper.getCurrentCpuUsage(); costhelper.getNetworkUsage(); } if (stream_type == StreamType.CloudRaw) { if (!Directory.Exists(exp_directory)) { Directory.CreateDirectory(exp_directory); } Logger logger = new Logger(); Byte[] val = new Byte[value.Length * num_operations]; // DateTime Date = new DateTime(DateTime.UtcNow.Ticks); // string cname = String.Format("CloudRaw-{0}", Date.ToString("yyyy-MM-dd")); // string bname = String.Format("{0}", Date.ToString("HH-mm-ss")); // string cname = String.Format("cloudraw-{0}", RandomString(4)); // string bname = String.Format("{0}", RandomString(4)); string cname = String.Format("cloudraw-{0}", RandName); string bname = String.Format("{0}", RandName); if (stream_op == StreamOperation.RandomKeyGet || stream_op == StreamOperation.RandomKeyGetMultipleSegments || stream_op == StreamOperation.RandomKeyGetAll) { doRawCloudPerf(val, SynchronizerType.Azure, SynchronizeDirection.Download, exp_directory, logger, containerName: cname, blobName: bname); logger.Dump(exp_directory + "/log"); } else { doRawCloudPerf(val, SynchronizerType.Azure, SynchronizeDirection.Upload, exp_directory, logger, containerName: cname, blobName: bname); logger.Dump(exp_directory + "/log"); } return; } if (stream_type == StreamType.DiskRaw) { if (!Directory.Exists(exp_directory)) { Directory.CreateDirectory(exp_directory); } Logger logger = doDiskRaw(stream_op, num_operations, value.Length, ptype, exp_directory); logger.Dump(exp_directory + "/log"); return; } // Are we getting raw disk throughput? if (stream_type == StreamType.Raw) { string ret = doDiskSpeed((value.Length * num_operations)/1000 + "K", value.Length/1000 + "K", rw); if (!Directory.Exists(exp_directory)) { Directory.CreateDirectory(exp_directory); } File.WriteAllText(exp_directory + "/log", ret); return; } // Populate the keys and the values Random random = new Random(DateTime.Now.Millisecond); StrKey[] keys = new StrKey[max_key]; for (int i = 0; i < max_key; ++i) { keys[i] = new StrKey("" + i); } /* List<ByteValue> vals = new List<ByteValue>(num_operations); Byte[][] tmp = new Byte[num_operations][]; for (int i = 0; i < num_operations; ++i) { tmp[i] = new Byte[value.Length]; random.NextBytes(tmp[i]); } for (int i = 0; i < num_operations; ++i) { keys[i] = new StrKey("" + i); vals.Add(new ByteValue(tmp[i])); // vals[i] = new ByteValue(tmp); } */ Logger log = new Logger(); // Open stream for different types of experiments if (stream_type == StreamType.Local && ptype == StreamFactory.StreamPhysicalType.File) { stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamPhysicalType.File) { stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamPhysicalType.File) { stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamPhysicalType.File) { stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.Local && ptype == StreamFactory.StreamPhysicalType.Directory) { stream = sf.openDirStream<StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamPhysicalType.Directory) { stream = sf.openDirStream<StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamPhysicalType.Directory) { stream = sf.openDirStream<StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamPhysicalType.Directory) { stream = sf.openDirStream<StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else { return; } if (stream_op == StreamOperation.RandomKeyRandomValueAppend) { List<ByteValue> vals = new List<ByteValue>(num_operations); Byte[][] tmp = new Byte[num_operations][]; for (int i = 0; i < num_operations; ++i) { tmp[i] = new Byte[value.Length]; random.NextBytes(tmp[i]); } for (int i = 0; i < num_operations; ++i) { vals.Add(new ByteValue(tmp[i])); } for (int i = 0; i < num_operations; ++i) { baselineStorageKV += keys[i].Size(); baselineStorageKV += vals[i].Size(); stream.Append(keys[i], vals[i]); } stream.Close(); } else if (stream_op == StreamOperation.RandomKeySameValueAppend) { Byte[] singleval = new Byte[value.Length]; random.NextBytes(singleval); ByteValue singlebv = new ByteValue(singleval); for (int i = 0; i < num_operations; ++i) { baselineStorageKV += keys[i].Size(); baselineStorageKV += value.Length; stream.Append(keys[i], singlebv); } stream.Close(); } else if (stream_op == StreamOperation.SameKeySameValueAppend) { StrKey key = new StrKey("ExpKey"); Byte[] singleval = new Byte[value.Length]; random.NextBytes(singleval); ByteValue singlebv = new ByteValue(singleval); for (int i = 0; i < num_operations; ++i) { stream.Append(key, singlebv); // System.Threading.Thread.Sleep(10); } stream.Close(); } else if (stream_op == StreamOperation.RandomKeyGet || stream_op == StreamOperation.RandomKeyGetMultipleSegments) { for (int i = 0; i < num_operations; ++i) { stream.Get(keys[random.Next(0, max_key)]); } stream.Close(); } else if (stream_op == StreamOperation.RandomKeyGetAll) { StrKey key = new StrKey("ExpKey"); for (int i = 0; i < num_operations; ) { long st = 0; long et = -1; Console.WriteLine(stime + ":" + etime); while (et < st) { st = RandomLong(stime, etime, random); // et = RandomLong(stime, etime, random); et = st + (10 * 10 * TimeSpan.TicksPerMillisecond); } Console.WriteLine(st + ":" + et); IEnumerable<IDataItem> iterator = stream.GetAll(key, st, et); foreach (IDataItem data in iterator) { data.GetVal(); ++i; if (i == num_operations) break; } } stream.Close(); } else if (stream_op == StreamOperation.SameKeyRandomValueAppend) { StrKey key = new StrKey("ExpKey"); for (int i = 0; i < num_operations; ++i) { baselineStorageKV += key.Size(); // baselineStorageKV += vals[i].Size(); // stream.Append(key, vals[i]); } stream.Close(); } else { for (int i = 0; i < num_operations; ++i) { stream.Get(new StrKey("" + random.Next(0,num_operations - 1))); } stream.Close(); } // Dump the instrumentation logs stream.DumpLogs(exp_directory + "/log"); // Collect costs usage List<string> costs = new List<string>(); if (doCosts) { costs.Add(DateTime.UtcNow.Ticks + ": CPU: " + costhelper.getCurrentCpuUsage()); costs.Add(DateTime.UtcNow.Ticks + ": Network: " + costhelper.getNetworkUsage()); costs.Add(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated:true)/1000.0f); costs.Add(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated:false)/1000.0f); costs.Add(DateTime.UtcNow.Ticks + ": Baseline Storage: " + baselineStorageKV/1000.0f); } File.AppendAllLines(exp_directory + "/log", costs); // sf.deleteStream(streamid, ci); }
public List<string> Dump(string CallerName, string HomeName, string AppName, string StreamName, StreamType stream_type, StreamOperation stream_op, StreamFactory.StreamPhysicalType ptype, Byte[] value, int num_operations, SynchronizerType synctype, bool doCosts= false) { List<string> exp_details = new List<string>(); exp_details.Add(exp_id); File.AppendAllLines(exp_directory + "/exp", exp_details); return exp_details; }
/* Sample call for upload:- byte[] array = new byte[1024*1024*1024]; Random random = new Random(); random.NextBytes(array); double timeTaken_Upload = Experiment.doRawCloudPerf(array, SynchronizerType.Azure, SynchronizeDirection.Upload, "fooContainer", "fooBlob"); double timeTaken_Download = Experiment.doRawCloudPerf(array, SynchronizerType.Azure, SynchronizeDirection.Download, "fooContainer", "fooBlob"); * * */ public static double doRawCloudPerf(byte[] input, SynchronizerType synchronizerType, SynchronizeDirection syncDirection, string exp_directory, Logger logger, string containerName=null, string blobName=null) { string accountName = ConfigurationManager.AppSettings.Get("AccountName"); string accountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey"); DateTime begin=DateTime.Now, end=DateTime.Now; if (synchronizerType == SynchronizerType.Azure) { #region azure download/upload if (containerName==null) containerName = "testingraw"; if(blobName==null) blobName = Guid.NewGuid().ToString(); CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentialsAccountAndKey(accountName, accountKey), true); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(containerName); if (syncDirection == SynchronizeDirection.Upload) { logger.Log("Start Stream Append"); container.CreateIfNotExist(); begin = DateTime.UtcNow;////////////////////////////////////// try { using (MemoryStream memoryStream = new System.IO.MemoryStream(input)) { CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName); blockBlob.UploadFromStream(memoryStream); } } catch (Exception e) { } end = DateTime.UtcNow;////////////////////////////////////// logger.Log("End Stream Append"); } if (syncDirection == SynchronizeDirection.Download) { logger.Log("Start Stream Get"); logger.Log("Start Stream GetAll"); try { CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName); byte[] blobContents = blockBlob.DownloadByteArray(); //if (File.Exists(blobName)) // File.Delete(blobName); begin = DateTime.UtcNow;////////////////////////////////////// // using (FileStream fs = new FileStream(blobName, FileMode.OpenOrCreate)) // { byte[] contents = blockBlob.DownloadByteArray(); // fs.Write(contents, 0, contents.Length); // } } catch (Exception e) { } end = DateTime.UtcNow;////////////////////////////////////// logger.Log("End Stream Get"); logger.Log("End Stream GetAll"); } #endregion } else if (synchronizerType == SynchronizerType.AmazonS3) { #region amazon s3 stuff if (containerName == null) containerName = "testingraw"; if (blobName == null) blobName = Guid.NewGuid().ToString(); AmazonS3Client amazonS3Client = new AmazonS3Client(accountName, accountKey); if (syncDirection == SynchronizeDirection.Upload) { ListBucketsResponse response = amazonS3Client.ListBuckets(); foreach (S3Bucket bucket in response.Buckets) { if (bucket.BucketName == containerName) { break; } } amazonS3Client.PutBucket(new PutBucketRequest().WithBucketName(containerName)); begin = DateTime.UtcNow;////////////////////////////////////// MemoryStream ms = new MemoryStream(); ms.Write(input, 0, input.Length); PutObjectRequest request = new PutObjectRequest(); request.WithBucketName(containerName); request.WithKey(blobName); request.InputStream = ms; amazonS3Client.PutObject(request); end = DateTime.UtcNow;////////////////////////////////////// } if (syncDirection == SynchronizeDirection.Download) { if (File.Exists(blobName)) File.Delete(blobName); begin = DateTime.UtcNow;////////////////////////////////////// GetObjectRequest request = new GetObjectRequest(); request.WithBucketName(containerName); request.WithKey(blobName); GetObjectResponse response = amazonS3Client.GetObject(request); var localFileStream = File.Create(blobName); response.ResponseStream.CopyTo(localFileStream); localFileStream.Close(); end = DateTime.UtcNow;////////////////////////////////////// } #endregion } else { throw new InvalidDataException("syncronizer type is not valid"); } return (end - begin).TotalMilliseconds;// return total time to upload in milliseconds }
public LocationInfo(string accName, string accKey, SynchronizerType St) { accountName = accName; accountKey = accKey; st = St; }
void Start() { if (type == SynchronizerType.ServerClientAuto) { type = Application.platform == RuntimePlatform.Android ? SynchronizerType.Client : SynchronizerType.Server; } if (type == SynchronizerType.Client) { if (File.Exists(HEADSET_PATH)) { using (var headsetStream = File.OpenRead(HEADSET_PATH)) { var reader = new BinaryReader(headsetStream); var num = reader.ReadInt32(); SetHeadsetNumber(num, false); } } clientSock = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); var ipEnd = new IPEndPoint(IPAddress.Any, BACKFILL_PORT); clientSock.Bind(ipEnd); clientSock.Listen(100); } if(type != SynchronizerType.Client && type != SynchronizerType.Playback) { // Start receiving real packets masterStream.StartReceiving(); } if (type == SynchronizerType.Recorder) { if (Directory.Exists(recordPath)) { var lastFrame = LastRecordedFrameNumber(); LoadRecordedFrame(lastFrame); } else { Directory.CreateDirectory(recordPath); } using (var s = File.OpenWrite(LastFrameNumberPath)) { var b = new BinaryWriter(s); b.Write(0u); } } }
public void Run(string CallerName, string HomeName, string AppName, string StreamName, string RandName, long stime, long etime, StreamType stream_type, StreamOperation stream_op, StreamFactory.StreamDataType ptype, CompressionType ctype, int ChunkSize, int ThreadPoolSize, Byte[] value, int num_operations, SynchronizerType synctype, int max_key = 0, string address = null, bool doCosts = false, bool doRaw = false) { // Set experiment directory CallerInfo ci = new CallerInfo(null, CallerName, CallerName, 1); exp_directory = Path.GetFullPath((null != ci.workingDir) ? ci.workingDir : Directory.GetCurrentDirectory()); exp_directory = exp_directory + "/" + HomeName + "/" + AppName + "/" + StreamName; if (max_key == 0) { max_key = num_operations; } // Set a description/tag for the experiment this.exp_id = "Directory: " + HomeName + "/" + AppName + "/" + StreamName + " Caller:" + CallerName + " Stream Type:" + stream_type + " Stream Op: " + stream_op + " Stream Ptype: " + ptype + " Compression Type: " + ctype + " Value size: " + value.Length + " num_operations: " + max_key + " actual_num_ops: " + num_operations + " Sync type: " + synctype + " Do costs? " + doCosts + "Chunk Size: " + ChunkSize + " ThreadPool Size:" + ThreadPoolSize; this.compressed_exp_id = " ST:" + stream_type + " OP: " + stream_op + " PT: " + ptype + " CT: " + ctype + " VS: " + value.Length + " I:" + num_operations + " MK:" + max_key + " SYNC: " + synctype + " chsize: " + ChunkSize + " nThreads: " + ThreadPoolSize; // Set remote storage server account info string AzureaccountName = ConfigurationManager.AppSettings.Get("AccountName"); string AzureaccountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey"); string S3accountName = ConfigurationManager.AppSettings.Get("S3AccountName"); string S3accountKey = ConfigurationManager.AppSettings.Get("S3AccountSharedKey"); LocationInfo Li; if (synctype == SynchronizerType.Azure) { Li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure); } else if (synctype == SynchronizerType.AmazonS3) { Li = new LocationInfo(S3accountName, S3accountKey, SynchronizerType.AmazonS3); } else { Li = null; } StreamFactory sf = StreamFactory.Instance; IStream stream = null; FqStreamID streamid = new FqStreamID(HomeName, AppName, StreamName); // Set op : R/W StreamFactory.StreamOp rw; if (stream_op == StreamOperation.RandomKeyRandomValueAppend || stream_op == StreamOperation.RandomKeySameValueAppend || stream_op == StreamOperation.SameKeyRandomValueAppend || stream_op == StreamOperation.SameKeySameValueAppend) { rw = StreamFactory.StreamOp.Write; } else { rw = StreamFactory.StreamOp.Read; } // Initialize costs CostsHelper costhelper = null; double baselineStorageKV = 0; if (doCosts) { costhelper = new CostsHelper(); costhelper.getCurrentCpuUsage(); costhelper.getNetworkUsage(); } if (stream_type == StreamType.CloudRaw) { if (!Directory.Exists(exp_directory)) { Directory.CreateDirectory(exp_directory); } Logger logger = new Logger(); Byte[] val = new Byte[value.Length * num_operations]; // DateTime Date = new DateTime(DateTime.UtcNow.Ticks); // string cname = String.Format("CloudRaw-{0}", Date.ToString("yyyy-MM-dd")); // string bname = String.Format("{0}", Date.ToString("HH-mm-ss")); // string cname = String.Format("cloudraw-{0}", RandomString(4)); // string bname = String.Format("{0}", RandomString(4)); string cname = String.Format("cloudraw-{0}", RandName); string bname = String.Format("{0}", RandName); if (stream_op == StreamOperation.RandomKeyGet || stream_op == StreamOperation.RandomKeyGetMultipleSegments || stream_op == StreamOperation.RandomKeyGetAll) { doRawCloudPerf(val, SynchronizerType.Azure, SynchronizeDirection.Download, exp_directory, logger, containerName: cname, blobName: bname); logger.Dump(exp_directory + "/log"); } else { doRawCloudPerf(val, SynchronizerType.Azure, SynchronizeDirection.Upload, exp_directory, logger, containerName: cname, blobName: bname); logger.Dump(exp_directory + "/log"); } return; } if (stream_type == StreamType.DiskRaw) { if (!Directory.Exists(exp_directory)) { Directory.CreateDirectory(exp_directory); } Logger logger = doDiskRaw(stream_op, num_operations, value.Length, ptype, exp_directory); logger.Dump(exp_directory + "/log"); return; } // Are we getting raw disk throughput? if (stream_type == StreamType.Raw) { string ret = doDiskSpeed((value.Length * num_operations) / 1000 + "K", value.Length / 1000 + "K", rw); if (!Directory.Exists(exp_directory)) { Directory.CreateDirectory(exp_directory); } File.WriteAllText(exp_directory + "/log", ret); return; } // Populate the keys and the values Random random = new Random(DateTime.Now.Millisecond); StrKey[] keys = new StrKey[max_key]; for (int i = 0; i < max_key; ++i) { keys[i] = new StrKey("" + i); } /* * List<ByteValue> vals = new List<ByteValue>(num_operations); * Byte[][] tmp = new Byte[num_operations][]; * for (int i = 0; i < num_operations; ++i) * { * tmp[i] = new Byte[value.Length]; * random.NextBytes(tmp[i]); * } * * for (int i = 0; i < num_operations; ++i) * { * keys[i] = new StrKey("" + i); * vals.Add(new ByteValue(tmp[i])); * // vals[i] = new ByteValue(tmp); * } */ Logger log = new Logger(); // Open stream for different types of experiments if (stream_type == StreamType.Local && ptype == StreamFactory.StreamDataType.Values) { stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamDataType.Values) { stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamDataType.Values) { stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamDataType.Values) { stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.Local && ptype == StreamFactory.StreamDataType.Files) { stream = sf.openFileDataStream <StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamDataType.Files) { stream = sf.openFileDataStream <StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamDataType.Files) { stream = sf.openFileDataStream <StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamDataType.Files) { stream = sf.openFileDataStream <StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else { return; } if (stream_op == StreamOperation.RandomKeyRandomValueAppend) { List <ByteValue> vals = new List <ByteValue>(num_operations); Byte[][] tmp = new Byte[num_operations][]; for (int i = 0; i < num_operations; ++i) { tmp[i] = new Byte[value.Length]; random.NextBytes(tmp[i]); } for (int i = 0; i < num_operations; ++i) { vals.Add(new ByteValue(tmp[i])); } for (int i = 0; i < num_operations; ++i) { baselineStorageKV += keys[i].Size(); baselineStorageKV += vals[i].Size(); stream.Append(keys[i], vals[i]); } stream.Close(); } else if (stream_op == StreamOperation.RandomKeySameValueAppend) { Byte[] singleval = new Byte[value.Length]; random.NextBytes(singleval); ByteValue singlebv = new ByteValue(singleval); for (int i = 0; i < num_operations; ++i) { baselineStorageKV += keys[i].Size(); baselineStorageKV += value.Length; stream.Append(keys[i], singlebv); } stream.Close(); } else if (stream_op == StreamOperation.SameKeySameValueAppend) { StrKey key = new StrKey("ExpKey"); Byte[] singleval = new Byte[value.Length]; random.NextBytes(singleval); ByteValue singlebv = new ByteValue(singleval); for (int i = 0; i < num_operations; ++i) { stream.Append(key, singlebv); // System.Threading.Thread.Sleep(10); } stream.Close(); } else if (stream_op == StreamOperation.RandomKeyGet || stream_op == StreamOperation.RandomKeyGetMultipleSegments) { for (int i = 0; i < num_operations; ++i) { stream.Get(keys[random.Next(0, max_key)]); } stream.Close(); } else if (stream_op == StreamOperation.RandomKeyGetAll) { StrKey key = new StrKey("ExpKey"); for (int i = 0; i < num_operations;) { long st = 0; long et = -1; Console.WriteLine(stime + ":" + etime); while (et < st) { st = RandomLong(stime, etime, random); // et = RandomLong(stime, etime, random); et = st + (10 * 10 * TimeSpan.TicksPerMillisecond); } Console.WriteLine(st + ":" + et); IEnumerable <IDataItem> iterator = stream.GetAll(key, st, et); foreach (IDataItem data in iterator) { data.GetVal(); ++i; if (i == num_operations) { break; } } } stream.Close(); } else if (stream_op == StreamOperation.SameKeyRandomValueAppend) { StrKey key = new StrKey("ExpKey"); for (int i = 0; i < num_operations; ++i) { baselineStorageKV += key.Size(); // baselineStorageKV += vals[i].Size(); // stream.Append(key, vals[i]); } stream.Close(); } else { for (int i = 0; i < num_operations; ++i) { stream.Get(new StrKey("" + random.Next(0, num_operations - 1))); } stream.Close(); } // Dump the instrumentation logs stream.DumpLogs(exp_directory + "/log"); // Collect costs usage List <string> costs = new List <string>(); if (doCosts) { costs.Add(DateTime.UtcNow.Ticks + ": CPU: " + costhelper.getCurrentCpuUsage()); costs.Add(DateTime.UtcNow.Ticks + ": Network: " + costhelper.getNetworkUsage()); costs.Add(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated: true) / 1000.0f); costs.Add(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated: false) / 1000.0f); costs.Add(DateTime.UtcNow.Ticks + ": Baseline Storage: " + baselineStorageKV / 1000.0f); } File.AppendAllLines(exp_directory + "/log", costs); // sf.deleteStream(streamid, ci); }
/* * Sample call for upload:- * byte[] array = new byte[1024*1024*1024]; * Random random = new Random(); * random.NextBytes(array); * double timeTaken_Upload = Experiment.doRawCloudPerf(array, SynchronizerType.Azure, SynchronizeDirection.Upload, "fooContainer", "fooBlob"); * double timeTaken_Download = Experiment.doRawCloudPerf(array, SynchronizerType.Azure, SynchronizeDirection.Download, "fooContainer", "fooBlob"); * * */ public static double doRawCloudPerf(byte[] input, SynchronizerType synchronizerType, SynchronizeDirection syncDirection, string exp_directory, Logger logger, string containerName = null, string blobName = null) { string accountName = ConfigurationManager.AppSettings.Get("AccountName"); string accountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey"); DateTime begin = DateTime.Now, end = DateTime.Now; if (synchronizerType == SynchronizerType.Azure) { #region azure download/upload if (containerName == null) { containerName = "testingraw"; } if (blobName == null) { blobName = Guid.NewGuid().ToString(); } CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentialsAccountAndKey(accountName, accountKey), true); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); CloudBlobContainer container = blobClient.GetContainerReference(containerName); if (syncDirection == SynchronizeDirection.Upload) { logger.Log("Start Stream Append"); container.CreateIfNotExist(); begin = DateTime.UtcNow;////////////////////////////////////// try { using (MemoryStream memoryStream = new System.IO.MemoryStream(input)) { CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName); blockBlob.UploadFromStream(memoryStream); } } catch (Exception e) { Console.WriteLine("exception: " + e); } end = DateTime.UtcNow;////////////////////////////////////// logger.Log("End Stream Append"); } if (syncDirection == SynchronizeDirection.Download) { logger.Log("Start Stream Get"); logger.Log("Start Stream GetAll"); try { CloudBlockBlob blockBlob = container.GetBlockBlobReference(blobName); byte[] blobContents = blockBlob.DownloadByteArray(); //if (File.Exists(blobName)) // File.Delete(blobName); begin = DateTime.UtcNow;////////////////////////////////////// // using (FileStream fs = new FileStream(blobName, FileMode.OpenOrCreate)) // { byte[] contents = blockBlob.DownloadByteArray(); // fs.Write(contents, 0, contents.Length); // } } catch (Exception e) { Console.WriteLine("exception: " + e); } end = DateTime.UtcNow;////////////////////////////////////// logger.Log("End Stream Get"); logger.Log("End Stream GetAll"); } #endregion } else if (synchronizerType == SynchronizerType.AmazonS3) { #region amazon s3 stuff if (containerName == null) { containerName = "testingraw"; } if (blobName == null) { blobName = Guid.NewGuid().ToString(); } AmazonS3Client amazonS3Client = new AmazonS3Client(accountName, accountKey); if (syncDirection == SynchronizeDirection.Upload) { ListBucketsResponse response = amazonS3Client.ListBuckets(); foreach (S3Bucket bucket in response.Buckets) { if (bucket.BucketName == containerName) { break; } } amazonS3Client.PutBucket(new PutBucketRequest().WithBucketName(containerName)); begin = DateTime.UtcNow;////////////////////////////////////// MemoryStream ms = new MemoryStream(); ms.Write(input, 0, input.Length); PutObjectRequest request = new PutObjectRequest(); request.WithBucketName(containerName); request.WithKey(blobName); request.InputStream = ms; amazonS3Client.PutObject(request); end = DateTime.UtcNow;////////////////////////////////////// } if (syncDirection == SynchronizeDirection.Download) { if (File.Exists(blobName)) { File.Delete(blobName); } begin = DateTime.UtcNow;////////////////////////////////////// GetObjectRequest request = new GetObjectRequest(); request.WithBucketName(containerName); request.WithKey(blobName); GetObjectResponse response = amazonS3Client.GetObject(request); var localFileStream = File.Create(blobName); response.ResponseStream.CopyTo(localFileStream); localFileStream.Close(); end = DateTime.UtcNow;////////////////////////////////////// } #endregion } else { throw new InvalidDataException("syncronizer type is not valid"); } return((end - begin).TotalMilliseconds);// return total time to upload in milliseconds }