private static void Write(IStream stream) { try { int i = 1; StrKey k1 = k1 = new StrKey("k1"); while (true) { stream.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-value" + i))); i++; Console.WriteLine("Written "+i+" values"); if (i %10==0) stream.Seal(false); if (isWriting) System.Threading.Thread.Sleep(1000); else break; } } catch(Exception e) { Console.WriteLine("Exception in write: "+e); } }
public NaivePreHeat_remoteread(string dataFilePath, int K, string outputFilePath, int chunkSize, string mdserver, LocationInfo li, int endSlotIndex ) { this.dataFilePath = dataFilePath; occupancyKey = new StrKey("occupancy"); random = new Random(); this.constK = K; this.outputFilePath = outputFilePath; this.chunkSize = chunkSize; this.mdserver = mdserver; this.li = li; int slotIndex = 0; StreamFactory streamFactory = StreamFactory.Instance; FqStreamID fq_sid = new FqStreamID(fqsidprefix + chunkSize, "A", "TestBS"); CallerInfo ci = new CallerInfo(null, "A", "A", 1); streamFactory.deleteStream(fq_sid, ci); IStream occupancyGroundTruthStream = streamFactory.openValueDataStream<StrKey, ByteValue>(fq_sid, ci, li, StreamFactory.StreamSecurityType.Plain, CompressionType.None, StreamFactory.StreamOp.Write, mdserver, chunkSize, 1, new Logger()); while (true) { occupancyGroundTruthStream.Append(occupancyKey, new ByteValue(BitConverter.GetBytes(random.Next(2))), slotIndexBase + slotIndex); slotIndex++; if (slotIndex == endSlotIndex) break; } occupancyGroundTruthStream.Close(); }
public void Setup() { StreamFactory sf = StreamFactory.Instance; DateTime Date = new DateTime(DateTime.UtcNow.Ticks); string HomeName = String.Format("TestHome-{0}", Date.ToString("yyyy-MM-dd")); string Caller = String.Format("{0}", Date.ToString("HH-mm-ss")); string AppName = Caller; Random rnd = new Random(); string StreamName = String.Format("{0}", rnd.Next()); FqStreamID fqstreamid = new FqStreamID(HomeName, AppName, StreamName); CallerInfo ci = new CallerInfo(null, Caller, Caller, 1); sf.deleteStream(fqstreamid, ci); vds = sf.openValueDataStream<StrKey, ByteValue>(fqstreamid, ci, null, StreamFactory.StreamSecurityType.Plain, CompressionType.None, StreamFactory.StreamOp.Write, mdserveraddress: "http://localhost:23456/MetadataServer"); k1 = new StrKey("k1"); k2 = new StrKey("k2"); }
private static void Read(IStream stream) { try { StrKey k1 = k1 = new StrKey("k1"); while (true) { IEnumerable<IDataItem> dataitems= stream.GetAll(k1); DateTime now = DateTime.Now; int count =0; foreach (IDataItem item in dataitems) { item.GetVal(); count++; } Console.WriteLine("[" + now + "]" + "GetAll "+count+" values received."); if (isReading) System.Threading.Thread.Sleep(ReadFrequencySeconds * 1000); else break; } } catch (Exception e) { Console.WriteLine("Exception in read: " + e); } }
public NaivePreHeat(string dataFilePath, int K, string outputFilePath) { this.dataFilePath = dataFilePath; occupancyKey = new StrKey("occupancy"); random = new Random(); this.constK = K; this.outputFilePath = outputFilePath; }
public void Setup() { k1 = new StrKey("k1"); k2 = new StrKey("k2"); string AzureaccountName = "testdrive"; string AzureaccountKey = "zRTT++dVryOWXJyAM7NM0TuQcu0Y23BgCQfkt7xh2f/Mm+r6c8/XtPTY0xxaF6tPSACJiuACsjotDeNIVyXM8Q=="; locationInfo = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure); }
public void Setup() { k1 = new StrKey("k1"); k2 = new StrKey("k2"); string AzureaccountName = "msrlot"; string AzureaccountKey = "wC3ou+VLBTu8ryKbsiSMtsIIHIpxGSIAWA0NtK58da2wELQ+USgYQwmVMtyq/p8ILsuZc8TiLeHBjaPI+a3A2Q=="; locationInfo = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure); }
public void TestDecodeEncode() { String seed = "SDJHRQF4GCMIIKAAAQ6IHY42X73FQFLHUULAPSKKD4DFDM7UXWWCRHBE"; byte[] secret = StrKey.DecodeCheck(VersionByte.Seed, seed); String encoded = StrKey.EncodeCheck(VersionByte.Seed, secret); Assert.AreEqual(seed, encoded); }
public void TestDecodeEncodeMuxedAccountWithLargeId() { var address = "MCAAAAAAAAAAAAB7BQ2L7E5NBWMXDUCMZSIPOBKRDSBYVLMXGSSKF6YNPIB7Y77ITKNOG"; var(id, key) = StrKey.DecodeStellarMuxedAccount(address); Assert.IsTrue(StrKey.IsValidMuxedAccount(address)); Assert.AreEqual(9223372036854775808UL, id); var encodedKey = StrKey.EncodeStellarAccountId(key); Assert.AreEqual("GA7QYNF7SOWQ3GLR2BGMZEHXAVIRZA4KVWLTJJFC7MGXUA74P7UJVSGZ", encodedKey); Assert.AreEqual(address, StrKey.EncodeStellarMuxedAccount(key, id)); }
public void Setup() { k1 = new StrKey("k1"); k2 = new StrKey("k2"); string AzureaccountName = "msrlot"; string AzureaccountKey = "wC3ou+VLBTu8ryKbsiSMtsIIHIpxGSIAWA0NtK58da2wELQ+USgYQwmVMtyq/p8ILsuZc8TiLeHBjaPI+a3A2Q=="; locationInfo = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure); streamID = new FqStreamID("99-a2000", "A0", "TestDS"); callerInfo = new CallerInfo(null, "A0", "A0", 1); streamSecurityType = StreamFactory.StreamSecurityType.Plain; streamFactory = StreamFactory.Instance; }
public void Setup() { k1 = new StrKey("k1"); k2 = new StrKey("k2"); string AzureaccountName = "testdrive"; string AzureaccountKey = "zRTT++dVryOWXJyAM7NM0TuQcu0Y23BgCQfkt7xh2f/Mm+r6c8/XtPTY0xxaF6tPSACJiuACsjotDeNIVyXM8Q=="; locationInfo = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure); streamID = new FqStreamID("99-a2000", "A0", "TestDS"); callerInfo = new CallerInfo(null, "A0", "A0", 1); streamSecurityType = StreamFactory.StreamSecurityType.Plain; streamFactory = StreamFactory.Instance; }
public bool IsAddressValid(string address) { try { StrKey.DecodeCheck(VersionByte.ed25519Publickey, address); return(true); } catch (Exception) { return(false); } }
public static KeyData GenerateKeyPairFromMnemonic(string mnemonic = null) { //Restore seed from word list var mo = String.IsNullOrWhiteSpace(mnemonic) ? new Mnemonic(Wordlist.English, WordCount.Twelve) : new Mnemonic(mnemonic, Wordlist.English); var extKey = mo.DeriveExtKey(); //var seed = extKey.PrivateKey.ToBytes(); var seed = StrKey.EncodeStellarSecretSeed(extKey.PrivateKey.ToBytes()); var kd = GenerateKeyPair(seed, mo); return(kd); }
public bool IsAddressValid(string address, out bool hasExtension) { hasExtension = false; bool containsForbiddenChar = false; if (string.IsNullOrWhiteSpace(address)) { return(false); } containsForbiddenChar = address.Any(_forbiddenAddressChars.Contains); if (containsForbiddenChar) { return(false); } var parts = address.Split(Constants.PublicAddressExtension.Separator, 2); try { var baseAddress = parts[0]; byte[] secret = StrKey.DecodeCheck(StrKey.VersionByte.ACCOUNT_ID, baseAddress); string encoded = StrKey.EncodeCheck(StrKey.VersionByte.ACCOUNT_ID, secret); if (baseAddress != encoded) { return(false); } } catch (Exception) { return(false); } if (parts.Length > 1) { if (!IsValidMemo(parts[1])) { return(false); } hasExtension = true; } return(true); }
private void AddPicDataStream(string key, byte[] imageBytes) { StrKey strKey = new StrKey(key); ByteValue byteVal = new ByteValue(imageBytes); try { lock (picStreamLock) { picStream.Append(strKey, byteVal); } } catch (Exception e) { logger.Log("Error while writing images to dir stream: {0}", e.ToString()); } }
private void AddTxtDataStream(string key, string message) { StrKey strKey = new StrKey(key); StrValue strVal = new StrValue(message); try { lock (textStreamLock) { textStream.Append(strKey, strVal); logger.Log("WaterAlert message has been written to {0}.", textStream.Get(strKey).ToString()); } } catch (Exception e) { logger.Log("Error while writing text to file stream: {0}", e.ToString()); } }
public override int GetHashCode() { int hash = 1; if (StrRecordName.Length != 0) { hash ^= StrRecordName.GetHashCode(); } if (StrKey.Length != 0) { hash ^= StrKey.GetHashCode(); } if (FieldVecList.Length != 0) { hash ^= FieldVecList.GetHashCode(); } if (ValueVecList.Length != 0) { hash ^= ValueVecList.GetHashCode(); } if (BExit != 0L) { hash ^= BExit.GetHashCode(); } if (Nreqid != 0L) { hash ^= Nreqid.GetHashCode(); } if (NRet != 0L) { hash ^= NRet.GetHashCode(); } if (EType != 0L) { hash ^= EType.GetHashCode(); } if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); } return(hash); }
public void Setup() { StreamFactory sf = StreamFactory.Instance; dfs_byte_val = sf.openValueDataStream <StrKey, ByteValue>(new FqStreamID("99-2729", "A0", "TestBS"), new CallerInfo(null, "A0", "A0", 1), null, StreamFactory.StreamSecurityType.Plain, CompressionType.None, StreamFactory.StreamOp.Write); dfs_str_val = sf.openValueDataStream <StrKey, StrValue>(new FqStreamID("99-2729", "A0", "TestSS"), new CallerInfo(null, "A0", "A0", 1), null, StreamFactory.StreamSecurityType.Plain, CompressionType.None, StreamFactory.StreamOp.Write); k1 = new StrKey("k1"); k2 = new StrKey("k2"); }
public static bool IsValid(StrKey.VersionByte versionByte, string encoded) { if (encoded?.Length != 56) { { return(false); } } try { var decoded = StrKey.DecodeCheck(versionByte, encoded); if (decoded.Length != 32) { { return(false); } } } catch { return(false); } return(true); }
public void Setup() { StreamFactory sf = StreamFactory.Instance; dfs_byte_val = sf.openValueDataStream<StrKey, ByteValue>(new FqStreamID("99-2729", "A0", "TestBS"), new CallerInfo(null, "A0", "A0", 1), null, StreamFactory.StreamSecurityType.Plain, CompressionType.None, StreamFactory.StreamOp.Write); dfs_str_val = sf.openValueDataStream<StrKey, StrValue>(new FqStreamID("99-2729", "A0", "TestSS"), new CallerInfo(null, "A0", "A0", 1), null, StreamFactory.StreamSecurityType.Plain, CompressionType.None, StreamFactory.StreamOp.Write); k1 = new StrKey("k1"); k2 = new StrKey("k2"); }
public async Task <IActionResult> EditPublicKey(string key) { if (!StrKey.IsValidEd25519PublicKey(key)) { ModelState.AddModelError(string.Empty, $"{key} is not a valid stellar key"); return(BadRequest(ModelState)); } string? userId = User.GetId(); ApplicationUser user = await _userManager.FindByIdAsync(userId); if (user == null) { ModelState.AddModelError(string.Empty, $"Could not load user with id {userId}"); return(BadRequest(ModelState)); } var claims = await _userManager.GetClaimsAsync(user); var profile = await _context.Profiles.FindAsync(user.Id); //ADD SEP-0010 PROTOCOL FOR VERIFYING OWNERSHIP if (key != profile.PublicKey) { profile.PublicKey = key; Claim keyClaim = claims.KeyClaim(); Claim newClaim = new Claim(Constants.Claims.PublicKey, key); if (keyClaim == null) { await _userManager.AddClaimAsync(user, newClaim); } else { await _userManager.ReplaceClaimAsync(user, keyClaim, newClaim); } _context.Profiles.Update(profile); _context.SaveChanges(); await _signInManager.RefreshSignInAsync(user); } return(Ok()); }
public void Setup() { StreamFactory sf = StreamFactory.Instance; DateTime Date = new DateTime(DateTime.UtcNow.Ticks); string HomeName = String.Format("TestHome-{0}", Date.ToString("yyyy-MM-dd")); string Caller = String.Format("{0}", Date.ToString("HH-mm-ss")); string AppName = Caller; Random rnd = new Random(); string StreamName = String.Format("{0}", rnd.Next()); FqStreamID fqstreamid = new FqStreamID(HomeName, AppName, StreamName); CallerInfo ci = new CallerInfo(null, Caller, Caller, 1); sf.deleteStream(fqstreamid, ci); vds = sf.openValueDataStream <StrKey, ByteValue>(fqstreamid, ci, null, StreamFactory.StreamSecurityType.Plain, CompressionType.None, StreamFactory.StreamOp.Write); k1 = new StrKey("k1"); k2 = new StrKey("k2"); }
public void TestDecodeInvalidVersionByte() { String accountId = "GCZHXL5HXQX5ABDM26LHYRCQZ5OJFHLOPLZX47WEBP3V2PF5AVFK2A5D"; var ex = Assert.Throws <FormatException>(() => StrKey.DecodeCheck(VersionByte.ed25519SecretSeed, accountId)); }
public void IsNotValidMuxedAccount(string address) { var result = StrKey.IsValidMuxedAccount(address); Assert.IsFalse(result); }
public static Logger doDiskRaw(StreamOperation op, int num_operations, int val_size, StreamFactory.StreamPhysicalType ptype, string exp_dir) { Logger logger = new Logger(); if (op == StreamOperation.RandomKeyRandomValueAppend || op == StreamOperation.SameKeyRandomValueAppend || op == StreamOperation.RandomKeySameValueAppend || op == StreamOperation.SameKeySameValueAppend) { // Populate the keys and the values Random random = new Random(DateTime.Now.Millisecond); StrKey[] keys = new StrKey[num_operations]; for (int i = 0; i < num_operations; ++i) { keys[i] = new StrKey("" + i); } Byte[] singleval = new Byte[val_size]; random.NextBytes(singleval); Byte[][] tmp = new Byte[num_operations][]; if (!(op == StreamOperation.RandomKeySameValueAppend || op == StreamOperation.SameKeySameValueAppend)) { for (int i = 0; i < num_operations; ++i) { tmp[i] = new Byte[val_size]; random.NextBytes(tmp[i]); } } StrKey key = new StrKey("ExpKey"); if (ptype == StreamFactory.StreamPhysicalType.File) { // ValueDataStream type logger.Log("Start Stream Open"); FileStream fout = new FileStream(exp_dir + "/DiskRaw", FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite); fout.Seek(0, SeekOrigin.End); BinaryWriter fs_bw = new BinaryWriter(fout); logger.Log("End Stream Open"); // TimeSpan span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0)); // Console.WriteLine("Time" + span.TotalSeconds); logger.Log("Start Stream Append"); for (int i = 0; i < num_operations; ++i) { fs_bw.BaseStream.Seek(0, SeekOrigin.End); /* if (op == StreamOperation.SameKeySameValueAppend || op == StreamOperation.SameKeyRandomValueAppend) { fs_bw.Write(key.key); } else { */ fs_bw.Write(keys[i].key); /* } */ fs_bw.Write(StreamFactory.NowUtc()); /* if (!(op == StreamOperation.RandomKeySameValueAppend || op == StreamOperation.SameKeySameValueAppend)) fs_bw.Write(tmp[i]); else */ fs_bw.Write(singleval); fs_bw.Flush(); } fout.Flush(true); fs_bw.Close(); // span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0)); // Console.WriteLine("Time" + span.TotalSeconds); logger.Log("End Stream Append"); } else { string[] filenames = new string[num_operations]; for (int i = 0; i < num_operations; ++i) { filenames[i] = exp_dir + "/DiskRaw" + i; } // FileDataStream type logger.Log("Start Stream Append"); for (int i = 0; i < num_operations; ++i) { FileStream fout = new FileStream(filenames[i], FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite); // fout.Seek(0, SeekOrigin.End); BinaryWriter fs_bw = new BinaryWriter(fout); // fs_bw.BaseStream.Seek(0, SeekOrigin.End); /* if (op == StreamOperation.SameKeySameValueAppend || op == StreamOperation.SameKeyRandomValueAppend) { fs_bw.Write(key.key); } else { */ fs_bw.Write(keys[i].key); // } fs_bw.Write(StreamFactory.NowUtc()); /* if (!(op == StreamOperation.RandomKeySameValueAppend || op == StreamOperation.SameKeySameValueAppend)) fs_bw.Write(tmp[i]); else */ fs_bw.Write(singleval); fout.Flush(true); fs_bw.Close(); } logger.Log("End Stream Append"); } } else { // Populate the keys and the values Random random = new Random(); Byte[] val = new Byte[val_size]; if (ptype == StreamFactory.StreamPhysicalType.File) { long[] offsets = new long[num_operations]; for (int i = 0; i < num_operations; ++i) { offsets[i] = random.Next(0, num_operations - 1) * val_size; } // ValueDataStream type logger.Log("Start Stream Open"); FileStream fin = new FileStream(exp_dir + "/DiskRaw", FileMode.Open, FileAccess.Read, FileShare.ReadWrite); BinaryReader fs_br = new BinaryReader(fin); logger.Log("End Stream Open"); logger.Log("Start Stream Get"); logger.Log("Start Stream GetAll"); for (int i = 0; i < num_operations; ++i) { fs_br.BaseStream.Seek(offsets[i], SeekOrigin.Begin); fs_br.Read(val, 0, val_size); } fs_br.Close(); logger.Log("End Stream Get"); logger.Log("End Stream GetAll"); } else { string[] filenames = new string[num_operations]; for (int i = 0; i < num_operations; ++i) { filenames[i] = exp_dir + "/DiskRaw" + random.Next(0, num_operations); } // FileDataStream type logger.Log("Start Stream Get"); logger.Log("Start Stream GetAll"); for (int i = 0; i < num_operations; ++i) { FileStream fin = new FileStream(filenames[i], FileMode.Open, FileAccess.Read, FileShare.ReadWrite); BinaryReader fs_br = new BinaryReader(fin); fs_br.Read(val, 0, val_size); fs_br.Close(); } logger.Log("End Stream Get"); logger.Log("End Stream GetAll"); } } CostsHelper costhelper = new CostsHelper(); logger.Log(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(exp_dir, dataRelated:true)/1000.0f); logger.Log(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(exp_dir, dataRelated:false)/1000.0f); return logger; }
static void Main(string[] args) { string AzureaccountName = ConfigurationManager.AppSettings.Get("AccountName"); string AzureaccountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey"); LocationInfo li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure); /* string dataFile = "D:\\b"; int KB = 1024; int[] chunk_sizes = { 4*1024*KB , 8*1024*KB }; for (int i = 1; i <= 1; i++) { for (int threads = 1; threads <= 1; threads++) { foreach (int csize in chunk_sizes) { Console.Write(">"); File.Copy(dataFile, dataFile + threads + "," + csize); AzureHelper helper = new AzureHelper(AzureaccountName, AzureaccountKey, "foo123123", CompressionType.None, EncryptionType.None, null, null, new Logger(), csize, threads); long start = DateTime.Now.Ticks; helper.UploadFileAsChunks(dataFile + threads + "," + csize); long end = DateTime.Now.Ticks; Console.WriteLine(threads + "," + csize + "," + (((double)(end - start) / (double)10000000)) ); } } } */ li = null; FqStreamID fq_sid = new FqStreamID("1299-2716", "A", "TestBS"); CallerInfo ci = new CallerInfo(null, "A", "A", 1); StreamFactory sf = StreamFactory.Instance; sf.deleteStream(fq_sid, ci); IStream dfs_byte_val = sf.openValueDataStream<StrKey, ByteValue>(fq_sid, ci, li, StreamFactory.StreamSecurityType.Plain, CompressionType.None, StreamFactory.StreamOp.Write); StrKey k1 = new StrKey("k1"); dfs_byte_val.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-cmu"))); dfs_byte_val.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-msr"))); dfs_byte_val.Seal(false); dfs_byte_val.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-uw"))); dfs_byte_val.Close(); Console.ReadKey(); dfs_byte_val = sf.openValueDataStream<StrKey, ByteValue>(fq_sid, ci, li, StreamFactory.StreamSecurityType.Plain, CompressionType.None, StreamFactory.StreamOp.Write); Console.WriteLine("Get in read : " + dfs_byte_val.Get(k1)); IEnumerable<IDataItem> data = dfs_byte_val.GetAll(k1, 0, StreamFactory.NowUtc()); foreach (IDataItem dataItem in data) Console.WriteLine(dataItem.GetVal().ToString()); dfs_byte_val.Close(); Console.ReadKey(); /* ValueSerializerBase<StrKey> vsb = new ValueSerializerBase<StrKey>(); Byte[] buffer1 = vsb.SerializeToByteStream().ToArray(); Byte[] buffer2 = SerializerHelper<StrKey>.SerializeToProtoStream(k1).ToArray(); FileStream fout = new FileStream("tmp.txt", FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite); BinaryWriter fs_bw = new BinaryWriter(fout); fs_bw.Write(buffer1); fs_bw.Write("-----W00t!-----"); fs_bw.Write(buffer2); fs_bw.Write("-----W00t!-----"); fs_bw.Close(); fout.Close(); */ }
public void TestDecodeInvalidVersionByte() { String accountId = "GCZHXL5HXQX5ABDM26LHYRCQZ5OJFHLOPLZX47WEBP3V2PF5AVFK2A5D"; StrKey.DecodeCheck(VersionByte.Seed, accountId); }
public void TestDecodeInvalidVersionByte() { var address = "GCZHXL5HXQX5ABDM26LHYRCQZ5OJFHLOPLZX47WEBP3V2PF5AVFK2A5D"; StrKey.DecodeCheck(StrKey.VersionByte.SEED, address); }
public static bool IsPublicKeyValid(string?publicKey) { return(StrKey.IsValidEd25519PublicKey(publicKey)); }
public void WriteToStream(string tag, string data) { StrKey key = new StrKey(tag); if (datastream != null) { datastream.Append(key, new StrValue(data)); } }
public void WriteToStream(string tag, string data) { //add the tag to a list if it's not already in there if (!tagList.Contains(tag)) { this.tagList.Add(tag); } StrKey key = new StrKey(tag); if (datastream != null) { datastream.Append(key, new StrValue(data)); logger.Log("Writing tag {0},{1} to stream ", key.ToString(), datastream.Get(key).ToString()); //Check if we should close it to force data to be written to Azure (AJB - remove once there is option to open stream with sycning at least every N minutes) double minutes = DateTime.Now.Subtract(streamClosed).TotalMinutes; if (minutes > 60) { streamClosed = DateTime.Now; datastream.Close(); logger.Log("{0}: closed and reopened data stream", streamClosed.ToString()); datastream = base.CreateFileStream<StrKey, StrValue>("data", true /* remoteSync */); } } }
private void AddTxtDataStream(string key, string message) { StrKey strKey = new StrKey(key); StrValue strVal = new StrValue(message); try { textStream = base.CreateFileStream<StrKey, StrValue>("H2OAlertsText", true); textStream.Append(strKey, strVal); logger.Log("WaterAlert message has been written to {0}.", textStream.Get(strKey).ToString()); textStream.Close(); } catch (Exception e) { logger.Log("Error while writing text to file stream: {0}", e.ToString()); } }
public void Run(string CallerName, string HomeName, string AppName, string StreamName, string RandName, long stime, long etime, StreamType stream_type, StreamOperation stream_op, StreamFactory.StreamDataType ptype, CompressionType ctype, int ChunkSize, int ThreadPoolSize, Byte[] value, int num_operations, SynchronizerType synctype, int max_key = 0, string address = null, bool doCosts = false, bool doRaw = false) { // Set experiment directory CallerInfo ci = new CallerInfo(null, CallerName, CallerName, 1); exp_directory = Path.GetFullPath((null != ci.workingDir) ? ci.workingDir : Directory.GetCurrentDirectory()); exp_directory = exp_directory + "/" + HomeName + "/" + AppName + "/" + StreamName; if (max_key == 0) { max_key = num_operations; } // Set a description/tag for the experiment this.exp_id = "Directory: " + HomeName + "/" + AppName + "/" + StreamName + " Caller:" + CallerName + " Stream Type:" + stream_type + " Stream Op: " + stream_op + " Stream Ptype: " + ptype + " Compression Type: " + ctype + " Value size: " + value.Length + " num_operations: " + max_key + " actual_num_ops: " + num_operations + " Sync type: " + synctype + " Do costs? " + doCosts + "Chunk Size: " + ChunkSize + " ThreadPool Size:" + ThreadPoolSize; this.compressed_exp_id = " ST:" + stream_type + " OP: " + stream_op + " PT: " + ptype + " CT: " + ctype + " VS: " + value.Length + " I:" + num_operations + " MK:" + max_key + " SYNC: " + synctype + " chsize: " + ChunkSize + " nThreads: " + ThreadPoolSize; // Set remote storage server account info string AzureaccountName = ConfigurationManager.AppSettings.Get("AccountName"); string AzureaccountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey"); string S3accountName = ConfigurationManager.AppSettings.Get("S3AccountName"); string S3accountKey = ConfigurationManager.AppSettings.Get("S3AccountSharedKey"); LocationInfo Li; if (synctype == SynchronizerType.Azure) { Li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure); } else if (synctype == SynchronizerType.AmazonS3) { Li = new LocationInfo(S3accountName, S3accountKey, SynchronizerType.AmazonS3); } else { Li = null; } StreamFactory sf = StreamFactory.Instance; IStream stream = null; FqStreamID streamid = new FqStreamID(HomeName, AppName, StreamName); // Set op : R/W StreamFactory.StreamOp rw; if (stream_op == StreamOperation.RandomKeyRandomValueAppend || stream_op == StreamOperation.RandomKeySameValueAppend || stream_op == StreamOperation.SameKeyRandomValueAppend || stream_op == StreamOperation.SameKeySameValueAppend) { rw = StreamFactory.StreamOp.Write; } else { rw = StreamFactory.StreamOp.Read; } // Initialize costs CostsHelper costhelper = null; double baselineStorageKV = 0; if (doCosts) { costhelper = new CostsHelper(); costhelper.getCurrentCpuUsage(); costhelper.getNetworkUsage(); } if (stream_type == StreamType.CloudRaw) { if (!Directory.Exists(exp_directory)) { Directory.CreateDirectory(exp_directory); } Logger logger = new Logger(); Byte[] val = new Byte[value.Length * num_operations]; // DateTime Date = new DateTime(DateTime.UtcNow.Ticks); // string cname = String.Format("CloudRaw-{0}", Date.ToString("yyyy-MM-dd")); // string bname = String.Format("{0}", Date.ToString("HH-mm-ss")); // string cname = String.Format("cloudraw-{0}", RandomString(4)); // string bname = String.Format("{0}", RandomString(4)); string cname = String.Format("cloudraw-{0}", RandName); string bname = String.Format("{0}", RandName); if (stream_op == StreamOperation.RandomKeyGet || stream_op == StreamOperation.RandomKeyGetMultipleSegments || stream_op == StreamOperation.RandomKeyGetAll) { doRawCloudPerf(val, SynchronizerType.Azure, SynchronizeDirection.Download, exp_directory, logger, containerName: cname, blobName: bname); logger.Dump(exp_directory + "/log"); } else { doRawCloudPerf(val, SynchronizerType.Azure, SynchronizeDirection.Upload, exp_directory, logger, containerName: cname, blobName: bname); logger.Dump(exp_directory + "/log"); } return; } if (stream_type == StreamType.DiskRaw) { if (!Directory.Exists(exp_directory)) { Directory.CreateDirectory(exp_directory); } Logger logger = doDiskRaw(stream_op, num_operations, value.Length, ptype, exp_directory); logger.Dump(exp_directory + "/log"); return; } // Are we getting raw disk throughput? if (stream_type == StreamType.Raw) { string ret = doDiskSpeed((value.Length * num_operations) / 1000 + "K", value.Length / 1000 + "K", rw); if (!Directory.Exists(exp_directory)) { Directory.CreateDirectory(exp_directory); } File.WriteAllText(exp_directory + "/log", ret); return; } // Populate the keys and the values Random random = new Random(DateTime.Now.Millisecond); StrKey[] keys = new StrKey[max_key]; for (int i = 0; i < max_key; ++i) { keys[i] = new StrKey("" + i); } /* * List<ByteValue> vals = new List<ByteValue>(num_operations); * Byte[][] tmp = new Byte[num_operations][]; * for (int i = 0; i < num_operations; ++i) * { * tmp[i] = new Byte[value.Length]; * random.NextBytes(tmp[i]); * } * * for (int i = 0; i < num_operations; ++i) * { * keys[i] = new StrKey("" + i); * vals.Add(new ByteValue(tmp[i])); * // vals[i] = new ByteValue(tmp); * } */ Logger log = new Logger(); // Open stream for different types of experiments if (stream_type == StreamType.Local && ptype == StreamFactory.StreamDataType.Values) { stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamDataType.Values) { stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamDataType.Values) { stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamDataType.Values) { stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.Local && ptype == StreamFactory.StreamDataType.Files) { stream = sf.openFileDataStream <StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamDataType.Files) { stream = sf.openFileDataStream <StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamDataType.Files) { stream = sf.openFileDataStream <StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamDataType.Files) { stream = sf.openFileDataStream <StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else { return; } if (stream_op == StreamOperation.RandomKeyRandomValueAppend) { List <ByteValue> vals = new List <ByteValue>(num_operations); Byte[][] tmp = new Byte[num_operations][]; for (int i = 0; i < num_operations; ++i) { tmp[i] = new Byte[value.Length]; random.NextBytes(tmp[i]); } for (int i = 0; i < num_operations; ++i) { vals.Add(new ByteValue(tmp[i])); } for (int i = 0; i < num_operations; ++i) { baselineStorageKV += keys[i].Size(); baselineStorageKV += vals[i].Size(); stream.Append(keys[i], vals[i]); } stream.Close(); } else if (stream_op == StreamOperation.RandomKeySameValueAppend) { Byte[] singleval = new Byte[value.Length]; random.NextBytes(singleval); ByteValue singlebv = new ByteValue(singleval); for (int i = 0; i < num_operations; ++i) { baselineStorageKV += keys[i].Size(); baselineStorageKV += value.Length; stream.Append(keys[i], singlebv); } stream.Close(); } else if (stream_op == StreamOperation.SameKeySameValueAppend) { StrKey key = new StrKey("ExpKey"); Byte[] singleval = new Byte[value.Length]; random.NextBytes(singleval); ByteValue singlebv = new ByteValue(singleval); for (int i = 0; i < num_operations; ++i) { stream.Append(key, singlebv); // System.Threading.Thread.Sleep(10); } stream.Close(); } else if (stream_op == StreamOperation.RandomKeyGet || stream_op == StreamOperation.RandomKeyGetMultipleSegments) { for (int i = 0; i < num_operations; ++i) { stream.Get(keys[random.Next(0, max_key)]); } stream.Close(); } else if (stream_op == StreamOperation.RandomKeyGetAll) { StrKey key = new StrKey("ExpKey"); for (int i = 0; i < num_operations;) { long st = 0; long et = -1; Console.WriteLine(stime + ":" + etime); while (et < st) { st = RandomLong(stime, etime, random); // et = RandomLong(stime, etime, random); et = st + (10 * 10 * TimeSpan.TicksPerMillisecond); } Console.WriteLine(st + ":" + et); IEnumerable <IDataItem> iterator = stream.GetAll(key, st, et); foreach (IDataItem data in iterator) { data.GetVal(); ++i; if (i == num_operations) { break; } } } stream.Close(); } else if (stream_op == StreamOperation.SameKeyRandomValueAppend) { StrKey key = new StrKey("ExpKey"); for (int i = 0; i < num_operations; ++i) { baselineStorageKV += key.Size(); // baselineStorageKV += vals[i].Size(); // stream.Append(key, vals[i]); } stream.Close(); } else { for (int i = 0; i < num_operations; ++i) { stream.Get(new StrKey("" + random.Next(0, num_operations - 1))); } stream.Close(); } // Dump the instrumentation logs stream.DumpLogs(exp_directory + "/log"); // Collect costs usage List <string> costs = new List <string>(); if (doCosts) { costs.Add(DateTime.UtcNow.Ticks + ": CPU: " + costhelper.getCurrentCpuUsage()); costs.Add(DateTime.UtcNow.Ticks + ": Network: " + costhelper.getNetworkUsage()); costs.Add(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated: true) / 1000.0f); costs.Add(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated: false) / 1000.0f); costs.Add(DateTime.UtcNow.Ticks + ": Baseline Storage: " + baselineStorageKV / 1000.0f); } File.AppendAllLines(exp_directory + "/log", costs); // sf.deleteStream(streamid, ci); }
public void TestDecodeInvalidSeed() { String seed = "SAA6NXOBOXP3RXGAXBW6PGFI5BPK4ODVAWITS4VDOMN5C2M4B66ZML"; var ex = Assert.Throws <FormatException>(() => StrKey.DecodeCheck(VersionByte.ed25519SecretSeed, seed)); }
public void WriteToStream(string imageData) // { StrKey key = new StrKey("ArduinoCamKey"); datastream.Append(key, new StrValue(imageData)); logger.Log("Writing imagedata to stream "); }
public void TestDecodeInvalidSeed() { var seed = "SAA6NXOBOXP3RXGAXBW6PGFI5BPK4ODVAWITS4VDOMN5C2M4B66ZML"; StrKey.DecodeCheck(StrKey.VersionByte.SEED, seed); }
static void Main(string[] args) { string AzureaccountName = ConfigurationManager.AppSettings.Get("AccountName"); string AzureaccountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey"); LocationInfo li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure); /* * string dataFile = "D:\\b"; * int KB = 1024; * int[] chunk_sizes = { 4*1024*KB , 8*1024*KB }; * * for (int i = 1; i <= 1; i++) * { * for (int threads = 1; threads <= 1; threads++) * { * foreach (int csize in chunk_sizes) * { * Console.Write(">"); * File.Copy(dataFile, dataFile + threads + "," + csize); * * AzureHelper helper = new AzureHelper(AzureaccountName, AzureaccountKey, "foo123123", CompressionType.None, EncryptionType.None, null, null, new Logger(), csize, threads); * long start = DateTime.Now.Ticks; * helper.UploadFileAsChunks(dataFile + threads + "," + csize); * long end = DateTime.Now.Ticks; * Console.WriteLine(threads + "," + csize + "," + (((double)(end - start) / (double)10000000)) ); * } * * } * } */ li = null; FqStreamID fq_sid = new FqStreamID("1299-2716", "A", "TestBS"); CallerInfo ci = new CallerInfo(null, "A", "A", 1); StreamFactory sf = StreamFactory.Instance; sf.deleteStream(fq_sid, ci); IStream dfs_byte_val = sf.openValueDataStream <StrKey, ByteValue>(fq_sid, ci, li, StreamFactory.StreamSecurityType.Plain, CompressionType.None, StreamFactory.StreamOp.Write); StrKey k1 = new StrKey("k1"); dfs_byte_val.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-cmu"))); dfs_byte_val.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-msr"))); dfs_byte_val.Seal(false); dfs_byte_val.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-uw"))); dfs_byte_val.Close(); Console.ReadKey(); dfs_byte_val = sf.openValueDataStream <StrKey, ByteValue>(fq_sid, ci, li, StreamFactory.StreamSecurityType.Plain, CompressionType.None, StreamFactory.StreamOp.Write); Console.WriteLine("Get in read : " + dfs_byte_val.Get(k1)); IEnumerable <IDataItem> data = dfs_byte_val.GetAll(k1, 0, StreamFactory.NowUtc()); foreach (IDataItem dataItem in data) { Console.WriteLine(dataItem.GetVal().ToString()); } dfs_byte_val.Close(); Console.ReadKey(); /* * ValueSerializerBase<StrKey> vsb = new ValueSerializerBase<StrKey>(); * Byte[] buffer1 = vsb.SerializeToByteStream().ToArray(); * Byte[] buffer2 = SerializerHelper<StrKey>.SerializeToProtoStream(k1).ToArray(); * * FileStream fout = new FileStream("tmp.txt", FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite); * BinaryWriter fs_bw = new BinaryWriter(fout); * fs_bw.Write(buffer1); * fs_bw.Write("-----W00t!-----"); * fs_bw.Write(buffer2); * fs_bw.Write("-----W00t!-----"); * fs_bw.Close(); * fout.Close(); */ }
public void Run(string CallerName, string HomeName, string AppName, string StreamName, string RandName, long stime, long etime, StreamType stream_type, StreamOperation stream_op, StreamFactory.StreamPhysicalType ptype, CompressionType ctype, int ChunkSize , int ThreadPoolSize, Byte[] value, int num_operations, SynchronizerType synctype, int max_key = 0, string address = null, bool doCosts= false, bool doRaw = false) { // Set experiment directory CallerInfo ci = new CallerInfo(null, CallerName, CallerName, 1); exp_directory = Path.GetFullPath((null != ci.workingDir) ? ci.workingDir : Directory.GetCurrentDirectory()); exp_directory = exp_directory + "/" + HomeName + "/" + AppName + "/" + StreamName; if (max_key == 0) max_key = num_operations; // Set a description/tag for the experiment this.exp_id = "Directory: " + HomeName + "/" + AppName + "/" + StreamName + " Caller:" + CallerName + " Stream Type:" + stream_type + " Stream Op: " + stream_op + " Stream Ptype: " + ptype + " Compression Type: " + ctype + " Value size: " + value.Length + " num_operations: " + max_key + " actual_num_ops: " + num_operations + " Sync type: " + synctype + " Do costs? " + doCosts + "Chunk Size: " + ChunkSize+ " ThreadPool Size:" +ThreadPoolSize; this.compressed_exp_id = " ST:" + stream_type + " OP: " + stream_op + " PT: " + ptype + " CT: " + ctype + " VS: " + value.Length + " I:" + num_operations + " MK:" + max_key + " SYNC: " + synctype+ " chsize: "+ChunkSize + " nThreads: "+ThreadPoolSize ; // Set remote storage server account info string AzureaccountName = ConfigurationManager.AppSettings.Get("AccountName"); string AzureaccountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey"); string S3accountName = ConfigurationManager.AppSettings.Get("S3AccountName"); string S3accountKey = ConfigurationManager.AppSettings.Get("S3AccountSharedKey"); LocationInfo Li; if (synctype == SynchronizerType.Azure) Li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure); else if (synctype == SynchronizerType.AmazonS3) Li = new LocationInfo(S3accountName, S3accountKey, SynchronizerType.AmazonS3); else Li = null; StreamFactory sf = StreamFactory.Instance; IStream stream = null; FqStreamID streamid = new FqStreamID(HomeName, AppName, StreamName); // Set op : R/W StreamFactory.StreamOp rw; if (stream_op == StreamOperation.RandomKeyRandomValueAppend || stream_op == StreamOperation.RandomKeySameValueAppend || stream_op == StreamOperation.SameKeyRandomValueAppend || stream_op == StreamOperation.SameKeySameValueAppend) { rw = StreamFactory.StreamOp.Write; } else { rw = StreamFactory.StreamOp.Read; } // Initialize costs CostsHelper costhelper = null; double baselineStorageKV = 0; if (doCosts) { costhelper = new CostsHelper(); costhelper.getCurrentCpuUsage(); costhelper.getNetworkUsage(); } if (stream_type == StreamType.CloudRaw) { if (!Directory.Exists(exp_directory)) { Directory.CreateDirectory(exp_directory); } Logger logger = new Logger(); Byte[] val = new Byte[value.Length * num_operations]; // DateTime Date = new DateTime(DateTime.UtcNow.Ticks); // string cname = String.Format("CloudRaw-{0}", Date.ToString("yyyy-MM-dd")); // string bname = String.Format("{0}", Date.ToString("HH-mm-ss")); // string cname = String.Format("cloudraw-{0}", RandomString(4)); // string bname = String.Format("{0}", RandomString(4)); string cname = String.Format("cloudraw-{0}", RandName); string bname = String.Format("{0}", RandName); if (stream_op == StreamOperation.RandomKeyGet || stream_op == StreamOperation.RandomKeyGetMultipleSegments || stream_op == StreamOperation.RandomKeyGetAll) { doRawCloudPerf(val, SynchronizerType.Azure, SynchronizeDirection.Download, exp_directory, logger, containerName: cname, blobName: bname); logger.Dump(exp_directory + "/log"); } else { doRawCloudPerf(val, SynchronizerType.Azure, SynchronizeDirection.Upload, exp_directory, logger, containerName: cname, blobName: bname); logger.Dump(exp_directory + "/log"); } return; } if (stream_type == StreamType.DiskRaw) { if (!Directory.Exists(exp_directory)) { Directory.CreateDirectory(exp_directory); } Logger logger = doDiskRaw(stream_op, num_operations, value.Length, ptype, exp_directory); logger.Dump(exp_directory + "/log"); return; } // Are we getting raw disk throughput? if (stream_type == StreamType.Raw) { string ret = doDiskSpeed((value.Length * num_operations)/1000 + "K", value.Length/1000 + "K", rw); if (!Directory.Exists(exp_directory)) { Directory.CreateDirectory(exp_directory); } File.WriteAllText(exp_directory + "/log", ret); return; } // Populate the keys and the values Random random = new Random(DateTime.Now.Millisecond); StrKey[] keys = new StrKey[max_key]; for (int i = 0; i < max_key; ++i) { keys[i] = new StrKey("" + i); } /* List<ByteValue> vals = new List<ByteValue>(num_operations); Byte[][] tmp = new Byte[num_operations][]; for (int i = 0; i < num_operations; ++i) { tmp[i] = new Byte[value.Length]; random.NextBytes(tmp[i]); } for (int i = 0; i < num_operations; ++i) { keys[i] = new StrKey("" + i); vals.Add(new ByteValue(tmp[i])); // vals[i] = new ByteValue(tmp); } */ Logger log = new Logger(); // Open stream for different types of experiments if (stream_type == StreamType.Local && ptype == StreamFactory.StreamPhysicalType.File) { stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamPhysicalType.File) { stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamPhysicalType.File) { stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamPhysicalType.File) { stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.Local && ptype == StreamFactory.StreamPhysicalType.Directory) { stream = sf.openDirStream<StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamPhysicalType.Directory) { stream = sf.openDirStream<StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamPhysicalType.Directory) { stream = sf.openDirStream<StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamPhysicalType.Directory) { stream = sf.openDirStream<StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log); } else { return; } if (stream_op == StreamOperation.RandomKeyRandomValueAppend) { List<ByteValue> vals = new List<ByteValue>(num_operations); Byte[][] tmp = new Byte[num_operations][]; for (int i = 0; i < num_operations; ++i) { tmp[i] = new Byte[value.Length]; random.NextBytes(tmp[i]); } for (int i = 0; i < num_operations; ++i) { vals.Add(new ByteValue(tmp[i])); } for (int i = 0; i < num_operations; ++i) { baselineStorageKV += keys[i].Size(); baselineStorageKV += vals[i].Size(); stream.Append(keys[i], vals[i]); } stream.Close(); } else if (stream_op == StreamOperation.RandomKeySameValueAppend) { Byte[] singleval = new Byte[value.Length]; random.NextBytes(singleval); ByteValue singlebv = new ByteValue(singleval); for (int i = 0; i < num_operations; ++i) { baselineStorageKV += keys[i].Size(); baselineStorageKV += value.Length; stream.Append(keys[i], singlebv); } stream.Close(); } else if (stream_op == StreamOperation.SameKeySameValueAppend) { StrKey key = new StrKey("ExpKey"); Byte[] singleval = new Byte[value.Length]; random.NextBytes(singleval); ByteValue singlebv = new ByteValue(singleval); for (int i = 0; i < num_operations; ++i) { stream.Append(key, singlebv); // System.Threading.Thread.Sleep(10); } stream.Close(); } else if (stream_op == StreamOperation.RandomKeyGet || stream_op == StreamOperation.RandomKeyGetMultipleSegments) { for (int i = 0; i < num_operations; ++i) { stream.Get(keys[random.Next(0, max_key)]); } stream.Close(); } else if (stream_op == StreamOperation.RandomKeyGetAll) { StrKey key = new StrKey("ExpKey"); for (int i = 0; i < num_operations; ) { long st = 0; long et = -1; Console.WriteLine(stime + ":" + etime); while (et < st) { st = RandomLong(stime, etime, random); // et = RandomLong(stime, etime, random); et = st + (10 * 10 * TimeSpan.TicksPerMillisecond); } Console.WriteLine(st + ":" + et); IEnumerable<IDataItem> iterator = stream.GetAll(key, st, et); foreach (IDataItem data in iterator) { data.GetVal(); ++i; if (i == num_operations) break; } } stream.Close(); } else if (stream_op == StreamOperation.SameKeyRandomValueAppend) { StrKey key = new StrKey("ExpKey"); for (int i = 0; i < num_operations; ++i) { baselineStorageKV += key.Size(); // baselineStorageKV += vals[i].Size(); // stream.Append(key, vals[i]); } stream.Close(); } else { for (int i = 0; i < num_operations; ++i) { stream.Get(new StrKey("" + random.Next(0,num_operations - 1))); } stream.Close(); } // Dump the instrumentation logs stream.DumpLogs(exp_directory + "/log"); // Collect costs usage List<string> costs = new List<string>(); if (doCosts) { costs.Add(DateTime.UtcNow.Ticks + ": CPU: " + costhelper.getCurrentCpuUsage()); costs.Add(DateTime.UtcNow.Ticks + ": Network: " + costhelper.getNetworkUsage()); costs.Add(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated:true)/1000.0f); costs.Add(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated:false)/1000.0f); costs.Add(DateTime.UtcNow.Ticks + ": Baseline Storage: " + baselineStorageKV/1000.0f); } File.AppendAllLines(exp_directory + "/log", costs); // sf.deleteStream(streamid, ci); }
public override string ToString() { return(StrKey.EncodeStellarAccountId(Data)); }
private void AddPicDataStream(string key, byte[] imageBytes) { StrKey strKey = new StrKey(key); ByteValue byteVal = new ByteValue(imageBytes); try { picStream = base.CreateDirStream<StrKey, ByteValue>("H2OAlertsPics", true); picStream.Append(strKey, byteVal); // logger.Log("WaterAlert picture has been written to {0}.", picStream.Get(strKey).ToString()); picStream.Close(); } catch (Exception e) { logger.Log("Error while writing images to dir stream: {0}", e.ToString()); } }
public void IsNotValidEd25519PublicKey(string address) { var result = StrKey.IsValidEd25519PublicKey(address); Assert.IsFalse(result); }
public void WriteToStream() { StrKey key = new StrKey("DummyKey"); datastream.Append(key, new StrValue("DummyVal")); logger.Log("Writing {0} to stream " , datastream.Get(key).ToString()); }
public static Logger doDiskRaw(StreamOperation op, int num_operations, int val_size, StreamFactory.StreamDataType ptype, string exp_dir) { Logger logger = new Logger(); if (op == StreamOperation.RandomKeyRandomValueAppend || op == StreamOperation.SameKeyRandomValueAppend || op == StreamOperation.RandomKeySameValueAppend || op == StreamOperation.SameKeySameValueAppend) { // Populate the keys and the values Random random = new Random(DateTime.Now.Millisecond); StrKey[] keys = new StrKey[num_operations]; for (int i = 0; i < num_operations; ++i) { keys[i] = new StrKey("" + i); } Byte[] singleval = new Byte[val_size]; random.NextBytes(singleval); Byte[][] tmp = new Byte[num_operations][]; if (!(op == StreamOperation.RandomKeySameValueAppend || op == StreamOperation.SameKeySameValueAppend)) { for (int i = 0; i < num_operations; ++i) { tmp[i] = new Byte[val_size]; random.NextBytes(tmp[i]); } } StrKey key = new StrKey("ExpKey"); if (ptype == StreamFactory.StreamDataType.Values) { // ValueDataStream type logger.Log("Start Stream Open"); FileStream fout = new FileStream(exp_dir + "/DiskRaw", FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite); fout.Seek(0, SeekOrigin.End); BinaryWriter fs_bw = new BinaryWriter(fout); logger.Log("End Stream Open"); // TimeSpan span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0)); // Console.WriteLine("Time" + span.TotalSeconds); logger.Log("Start Stream Append"); for (int i = 0; i < num_operations; ++i) { fs_bw.BaseStream.Seek(0, SeekOrigin.End); /* * if (op == StreamOperation.SameKeySameValueAppend || op == StreamOperation.SameKeyRandomValueAppend) * { * fs_bw.Write(key.key); * } * else * { */ fs_bw.Write(keys[i].key); /* * } */ fs_bw.Write(StreamFactory.NowUtc()); /* * if (!(op == StreamOperation.RandomKeySameValueAppend || * op == StreamOperation.SameKeySameValueAppend)) * fs_bw.Write(tmp[i]); * else */ fs_bw.Write(singleval); fs_bw.Flush(); } fout.Flush(true); fs_bw.Close(); // span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0)); // Console.WriteLine("Time" + span.TotalSeconds); logger.Log("End Stream Append"); } else { string[] filenames = new string[num_operations]; for (int i = 0; i < num_operations; ++i) { filenames[i] = exp_dir + "/DiskRaw" + i; } // FileDataStream type logger.Log("Start Stream Append"); for (int i = 0; i < num_operations; ++i) { FileStream fout = new FileStream(filenames[i], FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite); // fout.Seek(0, SeekOrigin.End); BinaryWriter fs_bw = new BinaryWriter(fout); // fs_bw.BaseStream.Seek(0, SeekOrigin.End); /* * if (op == StreamOperation.SameKeySameValueAppend || op == StreamOperation.SameKeyRandomValueAppend) * { * fs_bw.Write(key.key); * } * else * { */ fs_bw.Write(keys[i].key); // } fs_bw.Write(StreamFactory.NowUtc()); /* * if (!(op == StreamOperation.RandomKeySameValueAppend || * op == StreamOperation.SameKeySameValueAppend)) * fs_bw.Write(tmp[i]); * else */ fs_bw.Write(singleval); fout.Flush(true); fs_bw.Close(); } logger.Log("End Stream Append"); } } else { // Populate the keys and the values Random random = new Random(); Byte[] val = new Byte[val_size]; if (ptype == StreamFactory.StreamDataType.Values) { long[] offsets = new long[num_operations]; for (int i = 0; i < num_operations; ++i) { offsets[i] = random.Next(0, num_operations - 1) * val_size; } // ValueDataStream type logger.Log("Start Stream Open"); FileStream fin = new FileStream(exp_dir + "/DiskRaw", FileMode.Open, FileAccess.Read, FileShare.ReadWrite); BinaryReader fs_br = new BinaryReader(fin); logger.Log("End Stream Open"); logger.Log("Start Stream Get"); logger.Log("Start Stream GetAll"); for (int i = 0; i < num_operations; ++i) { fs_br.BaseStream.Seek(offsets[i], SeekOrigin.Begin); fs_br.Read(val, 0, val_size); } fs_br.Close(); logger.Log("End Stream Get"); logger.Log("End Stream GetAll"); } else { string[] filenames = new string[num_operations]; for (int i = 0; i < num_operations; ++i) { filenames[i] = exp_dir + "/DiskRaw" + random.Next(0, num_operations); } // FileDataStream type logger.Log("Start Stream Get"); logger.Log("Start Stream GetAll"); for (int i = 0; i < num_operations; ++i) { FileStream fin = new FileStream(filenames[i], FileMode.Open, FileAccess.Read, FileShare.ReadWrite); BinaryReader fs_br = new BinaryReader(fin); fs_br.Read(val, 0, val_size); fs_br.Close(); } logger.Log("End Stream Get"); logger.Log("End Stream GetAll"); } } CostsHelper costhelper = new CostsHelper(); logger.Log(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(exp_dir, dataRelated: true) / 1000.0f); logger.Log(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(exp_dir, dataRelated: false) / 1000.0f); return(logger); }