Ejemplo n.º 1
0
        private static void Write(IStream stream)
        {
            try
            {
                int i = 1;
                StrKey k1 = k1 = new StrKey("k1");
                while (true)
                {
                    stream.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-value" + i)));
                    i++;

                    Console.WriteLine("Written "+i+" values");
                   if (i %10==0)
                        stream.Seal(false);


                    if (isWriting)
                        System.Threading.Thread.Sleep(1000);
                    else
                        break;
                }
            }
            catch(Exception e)
            {
                Console.WriteLine("Exception in write: "+e);
            }

        }
        public NaivePreHeat_remoteread(string dataFilePath, int K, string outputFilePath, int chunkSize, string mdserver, LocationInfo li, int endSlotIndex )
        {
            this.dataFilePath = dataFilePath;
            occupancyKey = new StrKey("occupancy");
            random = new Random();
            this.constK = K;
            this.outputFilePath = outputFilePath;
            this.chunkSize = chunkSize;
            this.mdserver = mdserver;
            this.li = li;

            int slotIndex = 0;
            StreamFactory streamFactory = StreamFactory.Instance;
            FqStreamID fq_sid = new FqStreamID(fqsidprefix + chunkSize, "A", "TestBS");
            CallerInfo ci = new CallerInfo(null, "A", "A", 1);
            streamFactory.deleteStream(fq_sid, ci);
            IStream occupancyGroundTruthStream = streamFactory.openValueDataStream<StrKey, ByteValue>(fq_sid, ci, li, StreamFactory.StreamSecurityType.Plain, CompressionType.None, StreamFactory.StreamOp.Write, mdserver, chunkSize, 1, new Logger());
            while (true)
            {
                occupancyGroundTruthStream.Append(occupancyKey, new ByteValue(BitConverter.GetBytes(random.Next(2))), slotIndexBase + slotIndex);
                slotIndex++;
                if (slotIndex == endSlotIndex)
                    break;
            }
            occupancyGroundTruthStream.Close();
            

        }
        public void Setup()
        {
            StreamFactory sf = StreamFactory.Instance;
            DateTime Date = new DateTime(DateTime.UtcNow.Ticks);
            string HomeName = String.Format("TestHome-{0}", Date.ToString("yyyy-MM-dd"));
            string Caller = String.Format("{0}", Date.ToString("HH-mm-ss"));
            string AppName = Caller;
            Random rnd = new Random();
            string StreamName = String.Format("{0}", rnd.Next());

            FqStreamID fqstreamid = new FqStreamID(HomeName, AppName, StreamName);
            CallerInfo ci = new CallerInfo(null, Caller, Caller, 1);
            
            sf.deleteStream(fqstreamid, ci);
            
            vds = sf.openValueDataStream<StrKey, ByteValue>(fqstreamid, ci,
                                                                 null,
                                                                 StreamFactory.StreamSecurityType.Plain,
                                                                 CompressionType.None,
                                                                 StreamFactory.StreamOp.Write,
                                                                 mdserveraddress: "http://localhost:23456/MetadataServer");

            k1 = new StrKey("k1");
            k2 = new StrKey("k2");
        }
Ejemplo n.º 4
0
        private static void Read(IStream stream)
        {
            try
            {
                StrKey k1 = k1 = new StrKey("k1");
                while (true)
                {
                    IEnumerable<IDataItem> dataitems= stream.GetAll(k1);

                    DateTime now = DateTime.Now;
                    int count =0;
                    foreach (IDataItem item in dataitems)
                    {
                        item.GetVal();
                        count++;
                    }
                    
                    Console.WriteLine("[" + now + "]" + "GetAll "+count+" values received.");

                    if (isReading)
                        System.Threading.Thread.Sleep(ReadFrequencySeconds * 1000);
                    else
                        break;
                }
            }
            catch (Exception e)
            {
                Console.WriteLine("Exception in read: " + e);
            }

        }
Ejemplo n.º 5
0
 public NaivePreHeat(string dataFilePath, int K, string outputFilePath)
 {
     this.dataFilePath = dataFilePath;
     occupancyKey = new StrKey("occupancy");
     random = new Random();
     this.constK = K;
     this.outputFilePath = outputFilePath;
 }
 public void Setup()
 {
     k1 = new StrKey("k1");
     k2 = new StrKey("k2");
     string AzureaccountName = "testdrive";
     string AzureaccountKey = "zRTT++dVryOWXJyAM7NM0TuQcu0Y23BgCQfkt7xh2f/Mm+r6c8/XtPTY0xxaF6tPSACJiuACsjotDeNIVyXM8Q==";
     locationInfo = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
 }
Ejemplo n.º 7
0
 public void Setup()
 {
     k1 = new StrKey("k1");
     k2 = new StrKey("k2");
     string AzureaccountName = "msrlot";
     string AzureaccountKey = "wC3ou+VLBTu8ryKbsiSMtsIIHIpxGSIAWA0NtK58da2wELQ+USgYQwmVMtyq/p8ILsuZc8TiLeHBjaPI+a3A2Q==";
     locationInfo = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
 }
Ejemplo n.º 8
0
        public void TestDecodeEncode()
        {
            String seed = "SDJHRQF4GCMIIKAAAQ6IHY42X73FQFLHUULAPSKKD4DFDM7UXWWCRHBE";

            byte[] secret  = StrKey.DecodeCheck(VersionByte.Seed, seed);
            String encoded = StrKey.EncodeCheck(VersionByte.Seed, secret);

            Assert.AreEqual(seed, encoded);
        }
Ejemplo n.º 9
0
        public void TestDecodeEncodeMuxedAccountWithLargeId()
        {
            var address = "MCAAAAAAAAAAAAB7BQ2L7E5NBWMXDUCMZSIPOBKRDSBYVLMXGSSKF6YNPIB7Y77ITKNOG";

            var(id, key) = StrKey.DecodeStellarMuxedAccount(address);
            Assert.IsTrue(StrKey.IsValidMuxedAccount(address));
            Assert.AreEqual(9223372036854775808UL, id);
            var encodedKey = StrKey.EncodeStellarAccountId(key);

            Assert.AreEqual("GA7QYNF7SOWQ3GLR2BGMZEHXAVIRZA4KVWLTJJFC7MGXUA74P7UJVSGZ", encodedKey);
            Assert.AreEqual(address, StrKey.EncodeStellarMuxedAccount(key, id));
        }
Ejemplo n.º 10
0
 public void Setup()
 {
     k1 = new StrKey("k1");
     k2 = new StrKey("k2");
     string AzureaccountName = "msrlot";
     string AzureaccountKey = "wC3ou+VLBTu8ryKbsiSMtsIIHIpxGSIAWA0NtK58da2wELQ+USgYQwmVMtyq/p8ILsuZc8TiLeHBjaPI+a3A2Q==";
     locationInfo = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
     streamID = new FqStreamID("99-a2000", "A0", "TestDS");
     callerInfo = new CallerInfo(null, "A0", "A0", 1);
     streamSecurityType = StreamFactory.StreamSecurityType.Plain;
     streamFactory = StreamFactory.Instance;
 }
 public void Setup()
 {
     k1 = new StrKey("k1");
     k2 = new StrKey("k2");
     string AzureaccountName = "testdrive";
     string AzureaccountKey = "zRTT++dVryOWXJyAM7NM0TuQcu0Y23BgCQfkt7xh2f/Mm+r6c8/XtPTY0xxaF6tPSACJiuACsjotDeNIVyXM8Q==";
     locationInfo = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
     streamID = new FqStreamID("99-a2000", "A0", "TestDS");
     callerInfo = new CallerInfo(null, "A0", "A0", 1);
     streamSecurityType = StreamFactory.StreamSecurityType.Plain;
     streamFactory = StreamFactory.Instance;
 }
Ejemplo n.º 12
0
 public bool IsAddressValid(string address)
 {
     try
     {
         StrKey.DecodeCheck(VersionByte.ed25519Publickey, address);
         return(true);
     }
     catch (Exception)
     {
         return(false);
     }
 }
Ejemplo n.º 13
0
        public static KeyData GenerateKeyPairFromMnemonic(string mnemonic = null)
        {
            //Restore seed from word list
            var mo = String.IsNullOrWhiteSpace(mnemonic) ? new Mnemonic(Wordlist.English, WordCount.Twelve)
                                                   : new Mnemonic(mnemonic, Wordlist.English);
            var extKey = mo.DeriveExtKey();
            //var seed = extKey.PrivateKey.ToBytes();
            var seed = StrKey.EncodeStellarSecretSeed(extKey.PrivateKey.ToBytes());
            var kd   = GenerateKeyPair(seed, mo);

            return(kd);
        }
Ejemplo n.º 14
0
        public void Setup()
        {
            k1 = new StrKey("k1");
            k2 = new StrKey("k2");
            string AzureaccountName = "testdrive";
            string AzureaccountKey  = "zRTT++dVryOWXJyAM7NM0TuQcu0Y23BgCQfkt7xh2f/Mm+r6c8/XtPTY0xxaF6tPSACJiuACsjotDeNIVyXM8Q==";

            locationInfo       = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
            streamID           = new FqStreamID("99-a2000", "A0", "TestDS");
            callerInfo         = new CallerInfo(null, "A0", "A0", 1);
            streamSecurityType = StreamFactory.StreamSecurityType.Plain;
            streamFactory      = StreamFactory.Instance;
        }
        public bool IsAddressValid(string address, out bool hasExtension)
        {
            hasExtension = false;
            bool containsForbiddenChar = false;

            if (string.IsNullOrWhiteSpace(address))
            {
                return(false);
            }

            containsForbiddenChar = address.Any(_forbiddenAddressChars.Contains);

            if (containsForbiddenChar)
            {
                return(false);
            }

            var parts = address.Split(Constants.PublicAddressExtension.Separator, 2);

            try
            {
                var    baseAddress = parts[0];
                byte[] secret      = StrKey.DecodeCheck(StrKey.VersionByte.ACCOUNT_ID, baseAddress);
                string encoded     = StrKey.EncodeCheck(StrKey.VersionByte.ACCOUNT_ID, secret);

                if (baseAddress != encoded)
                {
                    return(false);
                }
            }
            catch (Exception)
            {
                return(false);
            }

            if (parts.Length > 1)
            {
                if (!IsValidMemo(parts[1]))
                {
                    return(false);
                }

                hasExtension = true;
            }

            return(true);
        }
Ejemplo n.º 16
0
        private void AddPicDataStream(string key, byte[] imageBytes)
        {
            StrKey    strKey  = new StrKey(key);
            ByteValue byteVal = new ByteValue(imageBytes);

            try
            {
                lock (picStreamLock)
                {
                    picStream.Append(strKey, byteVal);
                }
            }
            catch (Exception e)
            {
                logger.Log("Error while writing images to dir stream: {0}", e.ToString());
            }
        }
Ejemplo n.º 17
0
        private void AddTxtDataStream(string key, string message)
        {
            StrKey   strKey = new StrKey(key);
            StrValue strVal = new StrValue(message);

            try
            {
                lock (textStreamLock)
                {
                    textStream.Append(strKey, strVal);
                    logger.Log("WaterAlert message has been written to {0}.", textStream.Get(strKey).ToString());
                }
            }
            catch (Exception e)
            {
                logger.Log("Error while writing text to file stream: {0}", e.ToString());
            }
        }
Ejemplo n.º 18
0
        public override int GetHashCode()
        {
            int hash = 1;

            if (StrRecordName.Length != 0)
            {
                hash ^= StrRecordName.GetHashCode();
            }
            if (StrKey.Length != 0)
            {
                hash ^= StrKey.GetHashCode();
            }
            if (FieldVecList.Length != 0)
            {
                hash ^= FieldVecList.GetHashCode();
            }
            if (ValueVecList.Length != 0)
            {
                hash ^= ValueVecList.GetHashCode();
            }
            if (BExit != 0L)
            {
                hash ^= BExit.GetHashCode();
            }
            if (Nreqid != 0L)
            {
                hash ^= Nreqid.GetHashCode();
            }
            if (NRet != 0L)
            {
                hash ^= NRet.GetHashCode();
            }
            if (EType != 0L)
            {
                hash ^= EType.GetHashCode();
            }
            if (_unknownFields != null)
            {
                hash ^= _unknownFields.GetHashCode();
            }
            return(hash);
        }
Ejemplo n.º 19
0
        public void Setup()
        {
            StreamFactory sf = StreamFactory.Instance;

            dfs_byte_val = sf.openValueDataStream <StrKey, ByteValue>(new FqStreamID("99-2729", "A0", "TestBS"),
                                                                      new CallerInfo(null, "A0", "A0", 1),
                                                                      null,
                                                                      StreamFactory.StreamSecurityType.Plain,
                                                                      CompressionType.None,
                                                                      StreamFactory.StreamOp.Write);

            dfs_str_val = sf.openValueDataStream <StrKey, StrValue>(new FqStreamID("99-2729", "A0", "TestSS"),
                                                                    new CallerInfo(null, "A0", "A0", 1),
                                                                    null,
                                                                    StreamFactory.StreamSecurityType.Plain,
                                                                    CompressionType.None,
                                                                    StreamFactory.StreamOp.Write);

            k1 = new StrKey("k1");
            k2 = new StrKey("k2");
        }
Ejemplo n.º 20
0
        public static bool IsValid(StrKey.VersionByte versionByte, string encoded) 

        {
            if (encoded?.Length != 56)
            {
                
            {
                    
                return(false); 

                }
            }
            

 try
            {
                
 var decoded = StrKey.DecodeCheck(versionByte, encoded); 
                if (decoded.Length != 32)

                {
                    
                {
                        
                    return(false); 

                    }
                }
                

            } 
 catch
            {
                
                return(false); 

            } 
            return(true); 

        }
        public void Setup()
        {
            StreamFactory sf = StreamFactory.Instance;
            
            dfs_byte_val = sf.openValueDataStream<StrKey, ByteValue>(new FqStreamID("99-2729", "A0", "TestBS"),
                                                                 new CallerInfo(null, "A0", "A0", 1),
                                                                 null,
                                                                 StreamFactory.StreamSecurityType.Plain,
                                                                 CompressionType.None,
                                                                 StreamFactory.StreamOp.Write);

            dfs_str_val = sf.openValueDataStream<StrKey, StrValue>(new FqStreamID("99-2729", "A0", "TestSS"),
                                                                 new CallerInfo(null, "A0", "A0", 1),
                                                                 null,
                                                                 StreamFactory.StreamSecurityType.Plain,
                                                                 CompressionType.None,
                                                                 StreamFactory.StreamOp.Write);

            k1 = new StrKey("k1");
            k2 = new StrKey("k2");
        }
Ejemplo n.º 22
0
        public async Task <IActionResult> EditPublicKey(string key)
        {
            if (!StrKey.IsValidEd25519PublicKey(key))
            {
                ModelState.AddModelError(string.Empty, $"{key} is not a valid stellar key");
                return(BadRequest(ModelState));
            }
            string?         userId = User.GetId();
            ApplicationUser user   = await _userManager.FindByIdAsync(userId);

            if (user == null)
            {
                ModelState.AddModelError(string.Empty, $"Could not load user with id {userId}");
                return(BadRequest(ModelState));
            }
            var claims = await _userManager.GetClaimsAsync(user);

            var profile = await _context.Profiles.FindAsync(user.Id);

            //ADD SEP-0010 PROTOCOL FOR VERIFYING OWNERSHIP
            if (key != profile.PublicKey)
            {
                profile.PublicKey = key;
                Claim keyClaim = claims.KeyClaim();
                Claim newClaim = new Claim(Constants.Claims.PublicKey, key);

                if (keyClaim == null)
                {
                    await _userManager.AddClaimAsync(user, newClaim);
                }
                else
                {
                    await _userManager.ReplaceClaimAsync(user, keyClaim, newClaim);
                }
                _context.Profiles.Update(profile);
                _context.SaveChanges();
                await _signInManager.RefreshSignInAsync(user);
            }
            return(Ok());
        }
        public void Setup()
        {
            StreamFactory sf         = StreamFactory.Instance;
            DateTime      Date       = new DateTime(DateTime.UtcNow.Ticks);
            string        HomeName   = String.Format("TestHome-{0}", Date.ToString("yyyy-MM-dd"));
            string        Caller     = String.Format("{0}", Date.ToString("HH-mm-ss"));
            string        AppName    = Caller;
            Random        rnd        = new Random();
            string        StreamName = String.Format("{0}", rnd.Next());

            FqStreamID fqstreamid = new FqStreamID(HomeName, AppName, StreamName);
            CallerInfo ci         = new CallerInfo(null, Caller, Caller, 1);

            sf.deleteStream(fqstreamid, ci);

            vds = sf.openValueDataStream <StrKey, ByteValue>(fqstreamid, ci,
                                                             null,
                                                             StreamFactory.StreamSecurityType.Plain,
                                                             CompressionType.None,
                                                             StreamFactory.StreamOp.Write);

            k1 = new StrKey("k1");
            k2 = new StrKey("k2");
        }
Ejemplo n.º 24
0
 public void TestDecodeInvalidVersionByte()
 {
     String accountId = "GCZHXL5HXQX5ABDM26LHYRCQZ5OJFHLOPLZX47WEBP3V2PF5AVFK2A5D";
     var    ex        = Assert.Throws <FormatException>(() => StrKey.DecodeCheck(VersionByte.ed25519SecretSeed, accountId));
 }
Ejemplo n.º 25
0
        public void IsNotValidMuxedAccount(string address)
        {
            var result = StrKey.IsValidMuxedAccount(address);

            Assert.IsFalse(result);
        }
Ejemplo n.º 26
0
 private void AddPicDataStream(string key, byte[] imageBytes)
 {
     StrKey strKey = new StrKey(key);
     ByteValue byteVal = new ByteValue(imageBytes);
     try
     {
         lock (picStreamLock)
         {
             picStream.Append(strKey, byteVal);
         }
     }
     catch (Exception e)
     {
         logger.Log("Error while writing images to dir stream: {0}", e.ToString());
     }
 }
Ejemplo n.º 27
0
        public static Logger doDiskRaw(StreamOperation op, 
                    int num_operations, int val_size, 
                    StreamFactory.StreamPhysicalType ptype,
                    string exp_dir)
        {
            Logger logger = new Logger();
            if (op == StreamOperation.RandomKeyRandomValueAppend ||
                op == StreamOperation.SameKeyRandomValueAppend ||
                op == StreamOperation.RandomKeySameValueAppend ||
                op == StreamOperation.SameKeySameValueAppend)
            {

                // Populate the keys and the values
                Random random = new Random(DateTime.Now.Millisecond);
                StrKey[] keys = new StrKey[num_operations];
                for (int i = 0; i < num_operations; ++i)
                {
                    keys[i] = new StrKey("" + i);
                }

                Byte[] singleval = new Byte[val_size];
                random.NextBytes(singleval);
                Byte[][] tmp = new Byte[num_operations][];
                if (!(op == StreamOperation.RandomKeySameValueAppend ||
                    op == StreamOperation.SameKeySameValueAppend))
                {

                    for (int i = 0; i < num_operations; ++i)
                    {
                        tmp[i] = new Byte[val_size];
                        random.NextBytes(tmp[i]);
                    }
                }

                StrKey key = new StrKey("ExpKey");
                if (ptype == StreamFactory.StreamPhysicalType.File)
                {
                    // ValueDataStream type
                    logger.Log("Start Stream Open");
                    FileStream fout = new FileStream(exp_dir + "/DiskRaw", FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite);
                    fout.Seek(0, SeekOrigin.End);
                    BinaryWriter fs_bw = new BinaryWriter(fout);
                    logger.Log("End Stream Open");
                    // TimeSpan span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0));
                    // Console.WriteLine("Time" + span.TotalSeconds);
                    logger.Log("Start Stream Append");
                    for (int i = 0; i < num_operations; ++i)
                    {
                        fs_bw.BaseStream.Seek(0, SeekOrigin.End);
                        /*
                        if (op == StreamOperation.SameKeySameValueAppend || op == StreamOperation.SameKeyRandomValueAppend)
                        {
                            fs_bw.Write(key.key);
                        }
                        else
                        {
                        */
                            fs_bw.Write(keys[i].key);
                        /*
                        }
                        */
                        fs_bw.Write(StreamFactory.NowUtc());
                        /*
                        if (!(op == StreamOperation.RandomKeySameValueAppend ||
                            op == StreamOperation.SameKeySameValueAppend))
                            fs_bw.Write(tmp[i]);
                        else
                        */
                            fs_bw.Write(singleval);
                        fs_bw.Flush();
                    }
                    fout.Flush(true);
                    fs_bw.Close();
                    // span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0));
                    // Console.WriteLine("Time" + span.TotalSeconds);
                    logger.Log("End Stream Append");
                }
                else
                {
                    string[] filenames = new string[num_operations];
                    for (int i = 0; i < num_operations; ++i)
                    {
                        filenames[i] = exp_dir + "/DiskRaw" + i;
                    }
                    // FileDataStream type
                    logger.Log("Start Stream Append");
                    for (int i = 0; i < num_operations; ++i)
                    {
                        FileStream fout = new FileStream(filenames[i], FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite);
                        // fout.Seek(0, SeekOrigin.End);
                        BinaryWriter fs_bw = new BinaryWriter(fout);
                        // fs_bw.BaseStream.Seek(0, SeekOrigin.End);
                        /*
                        if (op == StreamOperation.SameKeySameValueAppend || op == StreamOperation.SameKeyRandomValueAppend)
                        {
                            fs_bw.Write(key.key);
                        }
                        else
                        {
                        */
                            fs_bw.Write(keys[i].key);
                        // }
                        fs_bw.Write(StreamFactory.NowUtc());
                        /*
                        if (!(op == StreamOperation.RandomKeySameValueAppend ||
                            op == StreamOperation.SameKeySameValueAppend))
                            fs_bw.Write(tmp[i]);
                        else
                        */
                            fs_bw.Write(singleval);
                        fout.Flush(true);
                        fs_bw.Close();
                    }
                    logger.Log("End Stream Append");
                }
            }
            else
            {
                // Populate the keys and the values
                Random random = new Random();
                Byte[] val = new Byte[val_size];

                if (ptype == StreamFactory.StreamPhysicalType.File)
                {
                    long[] offsets = new long[num_operations];
                    for (int i = 0; i < num_operations; ++i)
                    {
                        offsets[i] = random.Next(0, num_operations - 1) * val_size;
                    }
                    // ValueDataStream type
                    logger.Log("Start Stream Open");
                    FileStream fin = new FileStream(exp_dir + "/DiskRaw", FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
                    BinaryReader fs_br = new BinaryReader(fin);
                    logger.Log("End Stream Open");
                    logger.Log("Start Stream Get");
                    logger.Log("Start Stream GetAll");
                    for (int i = 0; i < num_operations; ++i)
                    {
                        fs_br.BaseStream.Seek(offsets[i], SeekOrigin.Begin);
                        fs_br.Read(val, 0, val_size);
                    }
                    fs_br.Close();
                    logger.Log("End Stream Get");
                    logger.Log("End Stream GetAll");
                }
                else
                {
                    string[] filenames = new string[num_operations];
                    for (int i = 0; i < num_operations; ++i)
                    {
                        filenames[i] = exp_dir + "/DiskRaw" + random.Next(0, num_operations);
                    }
                    // FileDataStream type
                    logger.Log("Start Stream Get");
                    logger.Log("Start Stream GetAll");
                    for (int i = 0; i < num_operations; ++i)
                    {
                        FileStream fin = new FileStream(filenames[i], FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
                        BinaryReader fs_br = new BinaryReader(fin);
                        fs_br.Read(val, 0, val_size);
                        fs_br.Close();
                    }
                    logger.Log("End Stream Get");
                    logger.Log("End Stream GetAll");
                }
            }
            CostsHelper costhelper = new CostsHelper();
            logger.Log(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(exp_dir, dataRelated:true)/1000.0f);
            logger.Log(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(exp_dir, dataRelated:false)/1000.0f);
            return logger;
        }
Ejemplo n.º 28
0
        static void Main(string[] args)
        {
            string AzureaccountName = ConfigurationManager.AppSettings.Get("AccountName");
            string AzureaccountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey");
            LocationInfo li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);

            /*
            string dataFile = "D:\\b";
            int KB = 1024; 
            int[] chunk_sizes = { 4*1024*KB , 8*1024*KB };

            for (int i = 1; i <= 1; i++)
            {
                for (int threads = 1; threads <= 1; threads++)
                {
                    foreach (int csize in chunk_sizes)
                    {
                        Console.Write(">");
                        File.Copy(dataFile, dataFile + threads + "," + csize);

                        AzureHelper helper = new AzureHelper(AzureaccountName, AzureaccountKey, "foo123123", CompressionType.None, EncryptionType.None, null, null, new Logger(), csize, threads);
                        long start = DateTime.Now.Ticks;
                        helper.UploadFileAsChunks(dataFile + threads + "," + csize);
                        long end = DateTime.Now.Ticks;
                        Console.WriteLine(threads + "," + csize + "," + (((double)(end - start) / (double)10000000)) );
                    }

                }
            }
            */










            li = null;
            FqStreamID fq_sid = new FqStreamID("1299-2716", "A", "TestBS");
            CallerInfo ci = new CallerInfo(null, "A", "A", 1);

            StreamFactory sf = StreamFactory.Instance;
            sf.deleteStream(fq_sid, ci);

            IStream dfs_byte_val = sf.openValueDataStream<StrKey, ByteValue>(fq_sid, ci, li,
                                                                 StreamFactory.StreamSecurityType.Plain,
                                                                 CompressionType.None,
                                                                 StreamFactory.StreamOp.Write);

            StrKey k1 = new StrKey("k1");

           dfs_byte_val.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-cmu")));
           dfs_byte_val.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-msr")));

           dfs_byte_val.Seal(false);
           dfs_byte_val.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-uw")));

            dfs_byte_val.Close();
            Console.ReadKey();

            
            dfs_byte_val = sf.openValueDataStream<StrKey, ByteValue>(fq_sid, ci, li,
                                                                 StreamFactory.StreamSecurityType.Plain,
                                                                 CompressionType.None,
                                                                 StreamFactory.StreamOp.Write);

            
            Console.WriteLine("Get in read : " + dfs_byte_val.Get(k1));
            IEnumerable<IDataItem> data = dfs_byte_val.GetAll(k1, 0, StreamFactory.NowUtc());
            foreach (IDataItem dataItem in data)
                Console.WriteLine(dataItem.GetVal().ToString());

            dfs_byte_val.Close();
            Console.ReadKey();

            /*
            ValueSerializerBase<StrKey> vsb = new ValueSerializerBase<StrKey>();
            Byte[] buffer1 = vsb.SerializeToByteStream().ToArray();
            Byte[] buffer2 = SerializerHelper<StrKey>.SerializeToProtoStream(k1).ToArray();

            FileStream fout = new FileStream("tmp.txt", FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite);
            BinaryWriter fs_bw = new BinaryWriter(fout);
            fs_bw.Write(buffer1);
            fs_bw.Write("-----W00t!-----");
            fs_bw.Write(buffer2);
            fs_bw.Write("-----W00t!-----");
            fs_bw.Close();
            fout.Close();
            */
        }
Ejemplo n.º 29
0
        public void TestDecodeInvalidVersionByte()
        {
            String accountId = "GCZHXL5HXQX5ABDM26LHYRCQZ5OJFHLOPLZX47WEBP3V2PF5AVFK2A5D";

            StrKey.DecodeCheck(VersionByte.Seed, accountId);
        }
Ejemplo n.º 30
0
        public void TestDecodeInvalidVersionByte()
        {
            var address = "GCZHXL5HXQX5ABDM26LHYRCQZ5OJFHLOPLZX47WEBP3V2PF5AVFK2A5D";

            StrKey.DecodeCheck(StrKey.VersionByte.SEED, address);
        }
Ejemplo n.º 31
0
 public static bool IsPublicKeyValid(string?publicKey)
 {
     return(StrKey.IsValidEd25519PublicKey(publicKey));
 }
Ejemplo n.º 32
0
        public void WriteToStream(string tag, string data)
        {

            StrKey key = new StrKey(tag);
            if (datastream != null)
            {
                datastream.Append(key, new StrValue(data));
                }
        }
Ejemplo n.º 33
0
        public void WriteToStream(string tag, string data)
        {
            //add the tag to a list if it's not already in there
            if (!tagList.Contains(tag))
            {
                this.tagList.Add(tag);
            }

            StrKey key = new StrKey(tag);
            if (datastream != null)
            {
                datastream.Append(key, new StrValue(data));
                logger.Log("Writing tag {0},{1} to stream ", key.ToString(), datastream.Get(key).ToString());

                //Check if we should close it to force data to be written to Azure (AJB - remove once there is option to open stream with sycning at least every N minutes)
                double minutes = DateTime.Now.Subtract(streamClosed).TotalMinutes;
                if (minutes > 60)
                {
                    streamClosed = DateTime.Now;
                    datastream.Close();
                    logger.Log("{0}: closed and reopened data stream", streamClosed.ToString());
                    datastream = base.CreateFileStream<StrKey, StrValue>("data", true /* remoteSync */);

                }

            }
        }
Ejemplo n.º 34
0
 private void AddTxtDataStream(string key, string message)
 {
     StrKey strKey = new StrKey(key);
     StrValue strVal = new StrValue(message);
     try
     {
         textStream = base.CreateFileStream<StrKey, StrValue>("H2OAlertsText", true);
         textStream.Append(strKey, strVal);
         logger.Log("WaterAlert message has been written to {0}.", textStream.Get(strKey).ToString());
         textStream.Close();
     }
     catch (Exception e)
     {
         logger.Log("Error while writing text to file stream: {0}", e.ToString());
     }
 }
Ejemplo n.º 35
0
        public void Run(string CallerName,
                        string HomeName, string AppName, string StreamName,
                        string RandName,
                        long stime, long etime,
                        StreamType stream_type,
                        StreamOperation stream_op,
                        StreamFactory.StreamDataType ptype,
                        CompressionType ctype, int ChunkSize, int ThreadPoolSize,
                        Byte[] value, int num_operations,
                        SynchronizerType synctype,
                        int max_key    = 0,
                        string address = null,
                        bool doCosts   = false,
                        bool doRaw     = false)
        {
            // Set experiment directory
            CallerInfo ci = new CallerInfo(null, CallerName, CallerName, 1);

            exp_directory = Path.GetFullPath((null != ci.workingDir) ? ci.workingDir : Directory.GetCurrentDirectory());
            exp_directory = exp_directory + "/" + HomeName + "/" + AppName + "/" + StreamName;

            if (max_key == 0)
            {
                max_key = num_operations;
            }

            // Set a description/tag for the experiment
            this.exp_id = "Directory: " + HomeName + "/" + AppName + "/" + StreamName +
                          " Caller:" + CallerName
                          + " Stream Type:" + stream_type + " Stream Op: " + stream_op + " Stream Ptype: " + ptype + " Compression Type: " + ctype
                          + " Value size: " + value.Length
                          + " num_operations: " + max_key
                          + " actual_num_ops: " + num_operations
                          + " Sync type: " + synctype
                          + " Do costs? " + doCosts + "Chunk Size: " + ChunkSize + " ThreadPool Size:" + ThreadPoolSize;

            this.compressed_exp_id =
                " ST:" + stream_type + " OP: " + stream_op + " PT: " + ptype + " CT: " + ctype
                + " VS: " + value.Length
                + " I:" + num_operations
                + " MK:" + max_key
                + " SYNC: " + synctype + " chsize: " + ChunkSize + " nThreads: " + ThreadPoolSize;

            // Set remote storage server account info
            string AzureaccountName = ConfigurationManager.AppSettings.Get("AccountName");
            string AzureaccountKey  = ConfigurationManager.AppSettings.Get("AccountSharedKey");

            string S3accountName = ConfigurationManager.AppSettings.Get("S3AccountName");
            string S3accountKey  = ConfigurationManager.AppSettings.Get("S3AccountSharedKey");

            LocationInfo Li;

            if (synctype == SynchronizerType.Azure)
            {
                Li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
            }
            else if (synctype == SynchronizerType.AmazonS3)
            {
                Li = new LocationInfo(S3accountName, S3accountKey, SynchronizerType.AmazonS3);
            }
            else
            {
                Li = null;
            }

            StreamFactory sf = StreamFactory.Instance;

            IStream    stream   = null;
            FqStreamID streamid = new FqStreamID(HomeName, AppName, StreamName);

            // Set op : R/W
            StreamFactory.StreamOp rw;
            if (stream_op == StreamOperation.RandomKeyRandomValueAppend ||
                stream_op == StreamOperation.RandomKeySameValueAppend ||
                stream_op == StreamOperation.SameKeyRandomValueAppend ||
                stream_op == StreamOperation.SameKeySameValueAppend)
            {
                rw = StreamFactory.StreamOp.Write;
            }
            else
            {
                rw = StreamFactory.StreamOp.Read;
            }

            // Initialize costs
            CostsHelper costhelper        = null;
            double      baselineStorageKV = 0;

            if (doCosts)
            {
                costhelper = new CostsHelper();
                costhelper.getCurrentCpuUsage();
                costhelper.getNetworkUsage();
            }

            if (stream_type == StreamType.CloudRaw)
            {
                if (!Directory.Exists(exp_directory))
                {
                    Directory.CreateDirectory(exp_directory);
                }
                Logger logger = new Logger();
                Byte[] val    = new Byte[value.Length * num_operations];
                // DateTime Date = new DateTime(DateTime.UtcNow.Ticks);
                // string cname = String.Format("CloudRaw-{0}", Date.ToString("yyyy-MM-dd"));
                // string bname = String.Format("{0}", Date.ToString("HH-mm-ss"));
                // string cname = String.Format("cloudraw-{0}", RandomString(4));
                // string bname = String.Format("{0}", RandomString(4));
                string cname = String.Format("cloudraw-{0}", RandName);
                string bname = String.Format("{0}", RandName);

                if (stream_op == StreamOperation.RandomKeyGet ||
                    stream_op == StreamOperation.RandomKeyGetMultipleSegments ||
                    stream_op == StreamOperation.RandomKeyGetAll)
                {
                    doRawCloudPerf(val, SynchronizerType.Azure,
                                   SynchronizeDirection.Download, exp_directory, logger, containerName: cname, blobName: bname);
                    logger.Dump(exp_directory + "/log");
                }
                else
                {
                    doRawCloudPerf(val, SynchronizerType.Azure,
                                   SynchronizeDirection.Upload, exp_directory, logger, containerName: cname, blobName: bname);
                    logger.Dump(exp_directory + "/log");
                }
                return;
            }

            if (stream_type == StreamType.DiskRaw)
            {
                if (!Directory.Exists(exp_directory))
                {
                    Directory.CreateDirectory(exp_directory);
                }
                Logger logger = doDiskRaw(stream_op, num_operations, value.Length, ptype, exp_directory);
                logger.Dump(exp_directory + "/log");
                return;
            }

            // Are we getting raw disk throughput?
            if (stream_type == StreamType.Raw)
            {
                string ret = doDiskSpeed((value.Length * num_operations) / 1000 + "K", value.Length / 1000 + "K", rw);
                if (!Directory.Exists(exp_directory))
                {
                    Directory.CreateDirectory(exp_directory);
                }
                File.WriteAllText(exp_directory + "/log", ret);
                return;
            }

            // Populate the keys and the values
            Random random = new Random(DateTime.Now.Millisecond);

            StrKey[] keys = new StrKey[max_key];
            for (int i = 0; i < max_key; ++i)
            {
                keys[i] = new StrKey("" + i);
            }

            /*
             * List<ByteValue> vals = new List<ByteValue>(num_operations);
             * Byte[][] tmp = new Byte[num_operations][];
             * for (int i = 0; i < num_operations; ++i)
             * {
             *  tmp[i] = new Byte[value.Length];
             *  random.NextBytes(tmp[i]);
             * }
             *
             * for (int i = 0; i < num_operations; ++i)
             * {
             *  keys[i] = new StrKey("" + i);
             *  vals.Add(new ByteValue(tmp[i]));
             *  // vals[i] = new ByteValue(tmp);
             * }
             */

            Logger log = new Logger();

            // Open stream for different types of experiments
            if (stream_type == StreamType.Local && ptype == StreamFactory.StreamDataType.Values)
            {
                stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }

            else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamDataType.Values)
            {
                stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }

            else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamDataType.Values)
            {
                stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }

            else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamDataType.Values)
            {
                stream = sf.openValueDataStream <StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }

            else if (stream_type == StreamType.Local && ptype == StreamFactory.StreamDataType.Files)
            {
                stream = sf.openFileDataStream <StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }

            else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamDataType.Files)
            {
                stream = sf.openFileDataStream <StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }
            else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamDataType.Files)
            {
                stream = sf.openFileDataStream <StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }
            else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamDataType.Files)
            {
                stream = sf.openFileDataStream <StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }
            else
            {
                return;
            }

            if (stream_op == StreamOperation.RandomKeyRandomValueAppend)
            {
                List <ByteValue> vals = new List <ByteValue>(num_operations);
                Byte[][]         tmp  = new Byte[num_operations][];
                for (int i = 0; i < num_operations; ++i)
                {
                    tmp[i] = new Byte[value.Length];
                    random.NextBytes(tmp[i]);
                }

                for (int i = 0; i < num_operations; ++i)
                {
                    vals.Add(new ByteValue(tmp[i]));
                }


                for (int i = 0; i < num_operations; ++i)
                {
                    baselineStorageKV += keys[i].Size();
                    baselineStorageKV += vals[i].Size();
                    stream.Append(keys[i], vals[i]);
                }
                stream.Close();
            }

            else if (stream_op == StreamOperation.RandomKeySameValueAppend)
            {
                Byte[] singleval = new Byte[value.Length];
                random.NextBytes(singleval);
                ByteValue singlebv = new ByteValue(singleval);
                for (int i = 0; i < num_operations; ++i)
                {
                    baselineStorageKV += keys[i].Size();
                    baselineStorageKV += value.Length;
                    stream.Append(keys[i], singlebv);
                }
                stream.Close();
            }

            else if (stream_op == StreamOperation.SameKeySameValueAppend)
            {
                StrKey key       = new StrKey("ExpKey");
                Byte[] singleval = new Byte[value.Length];
                random.NextBytes(singleval);
                ByteValue singlebv = new ByteValue(singleval);
                for (int i = 0; i < num_operations; ++i)
                {
                    stream.Append(key, singlebv);
                    // System.Threading.Thread.Sleep(10);
                }
                stream.Close();
            }

            else if (stream_op == StreamOperation.RandomKeyGet || stream_op == StreamOperation.RandomKeyGetMultipleSegments)
            {
                for (int i = 0; i < num_operations; ++i)
                {
                    stream.Get(keys[random.Next(0, max_key)]);
                }
                stream.Close();
            }

            else if (stream_op == StreamOperation.RandomKeyGetAll)
            {
                StrKey key = new StrKey("ExpKey");
                for (int i = 0; i < num_operations;)
                {
                    long st = 0;
                    long et = -1;
                    Console.WriteLine(stime + ":" + etime);
                    while (et < st)
                    {
                        st = RandomLong(stime, etime, random);
                        // et = RandomLong(stime, etime, random);
                        et = st + (10 * 10 * TimeSpan.TicksPerMillisecond);
                    }
                    Console.WriteLine(st + ":" + et);
                    IEnumerable <IDataItem> iterator = stream.GetAll(key, st, et);
                    foreach (IDataItem data in iterator)
                    {
                        data.GetVal();
                        ++i;

                        if (i == num_operations)
                        {
                            break;
                        }
                    }
                }
                stream.Close();
            }

            else if (stream_op == StreamOperation.SameKeyRandomValueAppend)
            {
                StrKey key = new StrKey("ExpKey");
                for (int i = 0; i < num_operations; ++i)
                {
                    baselineStorageKV += key.Size();
                    // baselineStorageKV += vals[i].Size();
                    // stream.Append(key, vals[i]);
                }
                stream.Close();
            }
            else
            {
                for (int i = 0; i < num_operations; ++i)
                {
                    stream.Get(new StrKey("" + random.Next(0, num_operations - 1)));
                }
                stream.Close();
            }

            // Dump the instrumentation logs
            stream.DumpLogs(exp_directory + "/log");

            // Collect costs usage
            List <string> costs = new List <string>();

            if (doCosts)
            {
                costs.Add(DateTime.UtcNow.Ticks + ": CPU: " + costhelper.getCurrentCpuUsage());
                costs.Add(DateTime.UtcNow.Ticks + ": Network: " + costhelper.getNetworkUsage());
                costs.Add(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated: true) / 1000.0f);
                costs.Add(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated: false) / 1000.0f);
                costs.Add(DateTime.UtcNow.Ticks + ": Baseline Storage: " + baselineStorageKV / 1000.0f);
            }
            File.AppendAllLines(exp_directory + "/log", costs);

            // sf.deleteStream(streamid, ci);
        }
Ejemplo n.º 36
0
 public void TestDecodeInvalidSeed()
 {
     String seed = "SAA6NXOBOXP3RXGAXBW6PGFI5BPK4ODVAWITS4VDOMN5C2M4B66ZML";
     var    ex   = Assert.Throws <FormatException>(() => StrKey.DecodeCheck(VersionByte.ed25519SecretSeed, seed));
 }
Ejemplo n.º 37
0
 private void AddTxtDataStream(string key, string message)
 {
     StrKey strKey = new StrKey(key);
     StrValue strVal = new StrValue(message);
     try
     {
         lock (textStreamLock)
         {
             textStream.Append(strKey, strVal);
             logger.Log("WaterAlert message has been written to {0}.", textStream.Get(strKey).ToString());
         }
     }
     catch (Exception e)
     {
         logger.Log("Error while writing text to file stream: {0}", e.ToString());
     }
 }
Ejemplo n.º 38
0
 public void WriteToStream(string imageData) //
 {
     StrKey key = new StrKey("ArduinoCamKey");
     datastream.Append(key, new StrValue(imageData));
     logger.Log("Writing imagedata to stream ");
 }
Ejemplo n.º 39
0
        public void TestDecodeInvalidSeed()
        {
            var seed = "SAA6NXOBOXP3RXGAXBW6PGFI5BPK4ODVAWITS4VDOMN5C2M4B66ZML";

            StrKey.DecodeCheck(StrKey.VersionByte.SEED, seed);
        }
Ejemplo n.º 40
0
        static void Main(string[] args)
        {
            string       AzureaccountName = ConfigurationManager.AppSettings.Get("AccountName");
            string       AzureaccountKey  = ConfigurationManager.AppSettings.Get("AccountSharedKey");
            LocationInfo li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);

            /*
             * string dataFile = "D:\\b";
             * int KB = 1024;
             * int[] chunk_sizes = { 4*1024*KB , 8*1024*KB };
             *
             * for (int i = 1; i <= 1; i++)
             * {
             *  for (int threads = 1; threads <= 1; threads++)
             *  {
             *      foreach (int csize in chunk_sizes)
             *      {
             *          Console.Write(">");
             *          File.Copy(dataFile, dataFile + threads + "," + csize);
             *
             *          AzureHelper helper = new AzureHelper(AzureaccountName, AzureaccountKey, "foo123123", CompressionType.None, EncryptionType.None, null, null, new Logger(), csize, threads);
             *          long start = DateTime.Now.Ticks;
             *          helper.UploadFileAsChunks(dataFile + threads + "," + csize);
             *          long end = DateTime.Now.Ticks;
             *          Console.WriteLine(threads + "," + csize + "," + (((double)(end - start) / (double)10000000)) );
             *      }
             *
             *  }
             * }
             */



            li = null;
            FqStreamID fq_sid = new FqStreamID("1299-2716", "A", "TestBS");
            CallerInfo ci     = new CallerInfo(null, "A", "A", 1);

            StreamFactory sf = StreamFactory.Instance;

            sf.deleteStream(fq_sid, ci);

            IStream dfs_byte_val = sf.openValueDataStream <StrKey, ByteValue>(fq_sid, ci, li,
                                                                              StreamFactory.StreamSecurityType.Plain,
                                                                              CompressionType.None,
                                                                              StreamFactory.StreamOp.Write);

            StrKey k1 = new StrKey("k1");

            dfs_byte_val.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-cmu")));
            dfs_byte_val.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-msr")));

            dfs_byte_val.Seal(false);
            dfs_byte_val.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-uw")));

            dfs_byte_val.Close();
            Console.ReadKey();


            dfs_byte_val = sf.openValueDataStream <StrKey, ByteValue>(fq_sid, ci, li,
                                                                      StreamFactory.StreamSecurityType.Plain,
                                                                      CompressionType.None,
                                                                      StreamFactory.StreamOp.Write);


            Console.WriteLine("Get in read : " + dfs_byte_val.Get(k1));
            IEnumerable <IDataItem> data = dfs_byte_val.GetAll(k1, 0, StreamFactory.NowUtc());

            foreach (IDataItem dataItem in data)
            {
                Console.WriteLine(dataItem.GetVal().ToString());
            }

            dfs_byte_val.Close();
            Console.ReadKey();

            /*
             * ValueSerializerBase<StrKey> vsb = new ValueSerializerBase<StrKey>();
             * Byte[] buffer1 = vsb.SerializeToByteStream().ToArray();
             * Byte[] buffer2 = SerializerHelper<StrKey>.SerializeToProtoStream(k1).ToArray();
             *
             * FileStream fout = new FileStream("tmp.txt", FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite);
             * BinaryWriter fs_bw = new BinaryWriter(fout);
             * fs_bw.Write(buffer1);
             * fs_bw.Write("-----W00t!-----");
             * fs_bw.Write(buffer2);
             * fs_bw.Write("-----W00t!-----");
             * fs_bw.Close();
             * fout.Close();
             */
        }
Ejemplo n.º 41
0
        public void Run(string CallerName,
                        string HomeName, string AppName, string StreamName, 
                        string RandName, 
                        long stime, long etime,
                        StreamType stream_type,
                        StreamOperation stream_op,
                        StreamFactory.StreamPhysicalType ptype,
                        CompressionType ctype, int ChunkSize , int ThreadPoolSize, 
                        Byte[] value, int num_operations,
                        SynchronizerType synctype,
                        int max_key = 0,
                        string address = null,
                        bool doCosts= false,
                        bool doRaw = false)
        {
            // Set experiment directory
            CallerInfo ci = new CallerInfo(null, CallerName, CallerName, 1);
            exp_directory = Path.GetFullPath((null != ci.workingDir) ? ci.workingDir : Directory.GetCurrentDirectory());
            exp_directory = exp_directory + "/" + HomeName + "/" + AppName + "/" + StreamName;

            if (max_key == 0)
                max_key = num_operations;

            // Set a description/tag for the experiment
            this.exp_id = "Directory: " + HomeName + "/" + AppName + "/" + StreamName +
                " Caller:" + CallerName
                + " Stream Type:" + stream_type + " Stream Op: " + stream_op + " Stream Ptype: " + ptype + " Compression Type: " + ctype
                + " Value size: " + value.Length
                + " num_operations: " + max_key
                + " actual_num_ops: " + num_operations
                + " Sync type: " + synctype
                + " Do costs? " + doCosts + "Chunk Size: " + ChunkSize+ " ThreadPool Size:" +ThreadPoolSize;

            this.compressed_exp_id =
                " ST:" + stream_type + " OP: " + stream_op + " PT: " + ptype + " CT: " + ctype
                + " VS: " + value.Length
                + " I:" + num_operations
                + " MK:" + max_key
                + " SYNC: " + synctype+ " chsize: "+ChunkSize + " nThreads: "+ThreadPoolSize  ;

            // Set remote storage server account info
            string AzureaccountName = ConfigurationManager.AppSettings.Get("AccountName");
            string AzureaccountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey");

            string S3accountName = ConfigurationManager.AppSettings.Get("S3AccountName");
            string S3accountKey = ConfigurationManager.AppSettings.Get("S3AccountSharedKey");

            LocationInfo Li;
            if (synctype == SynchronizerType.Azure)
                Li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
            else if (synctype == SynchronizerType.AmazonS3)
                Li = new LocationInfo(S3accountName, S3accountKey, SynchronizerType.AmazonS3);
            else
                Li = null;

            StreamFactory sf = StreamFactory.Instance;

            IStream stream = null;
            FqStreamID streamid = new FqStreamID(HomeName, AppName, StreamName);

            // Set op : R/W
            StreamFactory.StreamOp rw;
            if (stream_op == StreamOperation.RandomKeyRandomValueAppend
                || stream_op == StreamOperation.RandomKeySameValueAppend
                || stream_op == StreamOperation.SameKeyRandomValueAppend
                || stream_op == StreamOperation.SameKeySameValueAppend)
            {
                rw = StreamFactory.StreamOp.Write;
            }
            else
            {
                rw = StreamFactory.StreamOp.Read;
            }

            // Initialize costs
            CostsHelper costhelper = null;
            double baselineStorageKV = 0;
            if (doCosts)
            {
                costhelper = new CostsHelper();
                costhelper.getCurrentCpuUsage();
                costhelper.getNetworkUsage();
            }

            if (stream_type == StreamType.CloudRaw)
            {
                if (!Directory.Exists(exp_directory))
                {
                    Directory.CreateDirectory(exp_directory);
                }
                Logger logger = new Logger();
                Byte[] val = new Byte[value.Length * num_operations];
                // DateTime Date = new DateTime(DateTime.UtcNow.Ticks);
                // string cname = String.Format("CloudRaw-{0}", Date.ToString("yyyy-MM-dd"));
                // string bname = String.Format("{0}", Date.ToString("HH-mm-ss"));
                // string cname = String.Format("cloudraw-{0}", RandomString(4));
                // string bname = String.Format("{0}", RandomString(4));
                string cname = String.Format("cloudraw-{0}", RandName);
                string bname = String.Format("{0}", RandName);

                if (stream_op == StreamOperation.RandomKeyGet ||
                    stream_op == StreamOperation.RandomKeyGetMultipleSegments ||
                    stream_op == StreamOperation.RandomKeyGetAll)
                {
                    doRawCloudPerf(val, SynchronizerType.Azure,
                        SynchronizeDirection.Download, exp_directory, logger, containerName: cname, blobName: bname);
                    logger.Dump(exp_directory + "/log");
                }
                else
                {
                    doRawCloudPerf(val, SynchronizerType.Azure,
                        SynchronizeDirection.Upload, exp_directory, logger, containerName: cname, blobName: bname);
                    logger.Dump(exp_directory + "/log");
                }
                return;
            }

            if (stream_type == StreamType.DiskRaw)
            {
                if (!Directory.Exists(exp_directory))
                {
                    Directory.CreateDirectory(exp_directory);
                }
                Logger logger = doDiskRaw(stream_op, num_operations, value.Length, ptype, exp_directory);
                logger.Dump(exp_directory + "/log");
                return;
            }

            // Are we getting raw disk throughput?
            if (stream_type == StreamType.Raw)
            {
                string ret = doDiskSpeed((value.Length * num_operations)/1000 + "K", value.Length/1000 + "K", rw);
                if (!Directory.Exists(exp_directory))
                {
                    Directory.CreateDirectory(exp_directory);
                }
                File.WriteAllText(exp_directory + "/log", ret);
                return;
            }

            // Populate the keys and the values
            Random random = new Random(DateTime.Now.Millisecond);
            StrKey[] keys = new StrKey[max_key];
            for (int i = 0; i < max_key; ++i)
            {
                keys[i] = new StrKey("" + i);
            }

            /*
            List<ByteValue> vals = new List<ByteValue>(num_operations);
            Byte[][] tmp = new Byte[num_operations][];
            for (int i = 0; i < num_operations; ++i)
            {
                tmp[i] = new Byte[value.Length];
                random.NextBytes(tmp[i]);
            }

            for (int i = 0; i < num_operations; ++i)
            {
                keys[i] = new StrKey("" + i);
                vals.Add(new ByteValue(tmp[i]));
                // vals[i] = new ByteValue(tmp);
            }
            */

            Logger log = new Logger();
            // Open stream for different types of experiments
            if (stream_type == StreamType.Local && ptype == StreamFactory.StreamPhysicalType.File)
            {
                stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }

            else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamPhysicalType.File)
            {
                stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }

            else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamPhysicalType.File)
            {
                stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }

            else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamPhysicalType.File)
            {
                stream = sf.openFileStream<StrKey, ByteValue>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }

            else if (stream_type == StreamType.Local && ptype == StreamFactory.StreamPhysicalType.Directory)
            {
                stream = sf.openDirStream<StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }

            else if (stream_type == StreamType.LocalEnc && ptype == StreamFactory.StreamPhysicalType.Directory)
            {
                stream = sf.openDirStream<StrKey>(streamid, ci, null, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }
            else if (stream_type == StreamType.Remote && ptype == StreamFactory.StreamPhysicalType.Directory)
            {
                stream = sf.openDirStream<StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Plain, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }
            else if (stream_type == StreamType.RemoteEnc && ptype == StreamFactory.StreamPhysicalType.Directory)
            {
                stream = sf.openDirStream<StrKey>(streamid, ci, Li, StreamFactory.StreamSecurityType.Secure, ctype, rw, address, ChunkSize, ThreadPoolSize, log);
            }
            else
            {
                return;
            }

            if (stream_op == StreamOperation.RandomKeyRandomValueAppend)
            {
                List<ByteValue> vals = new List<ByteValue>(num_operations);
                Byte[][] tmp = new Byte[num_operations][];
                for (int i = 0; i < num_operations; ++i)
                {
                    tmp[i] = new Byte[value.Length];
                    random.NextBytes(tmp[i]);
                }

                for (int i = 0; i < num_operations; ++i)
                {
                    vals.Add(new ByteValue(tmp[i]));
                }

                for (int i = 0; i < num_operations; ++i)
                {
                    baselineStorageKV += keys[i].Size();
                    baselineStorageKV += vals[i].Size();
                    stream.Append(keys[i], vals[i]);
                }
                stream.Close();
            }

            else if (stream_op == StreamOperation.RandomKeySameValueAppend)
            {
                Byte[] singleval = new Byte[value.Length];
                random.NextBytes(singleval);
                ByteValue singlebv = new ByteValue(singleval);
                for (int i = 0; i < num_operations; ++i)
                {
                    baselineStorageKV += keys[i].Size();
                    baselineStorageKV += value.Length;
                    stream.Append(keys[i], singlebv);
                }
                stream.Close();
            }

            else if (stream_op == StreamOperation.SameKeySameValueAppend)
            {
                StrKey key = new StrKey("ExpKey");
                Byte[] singleval = new Byte[value.Length];
                random.NextBytes(singleval);
                ByteValue singlebv = new ByteValue(singleval);
                for (int i = 0; i < num_operations; ++i)
                {
                    stream.Append(key, singlebv);
                    // System.Threading.Thread.Sleep(10);
                }
                stream.Close();
            }

            else if (stream_op == StreamOperation.RandomKeyGet || stream_op == StreamOperation.RandomKeyGetMultipleSegments)
            {
                for (int i = 0; i < num_operations; ++i)
                {
                    stream.Get(keys[random.Next(0, max_key)]);
                }
                stream.Close();
            }

            else if (stream_op == StreamOperation.RandomKeyGetAll)
            {
                StrKey key = new StrKey("ExpKey");
                for (int i = 0; i < num_operations; )
                {
                    long st = 0;
                    long et = -1;
                    Console.WriteLine(stime + ":" + etime);
                    while (et < st)
                    {
                        st = RandomLong(stime, etime, random);
                        // et = RandomLong(stime, etime, random);
                        et = st + (10 * 10 * TimeSpan.TicksPerMillisecond);
                    }
                    Console.WriteLine(st + ":" + et);
                    IEnumerable<IDataItem> iterator = stream.GetAll(key, st, et);
                    foreach (IDataItem data in iterator)
                    {
                        data.GetVal();
                        ++i;

                        if (i == num_operations)
                            break;
                    }
                }
                stream.Close();
            }

            else if (stream_op == StreamOperation.SameKeyRandomValueAppend)
            {
                StrKey key = new StrKey("ExpKey");
                for (int i = 0; i < num_operations; ++i)
                {
                    baselineStorageKV += key.Size();
                    // baselineStorageKV += vals[i].Size();
                    // stream.Append(key, vals[i]);
                }
                stream.Close();
            }
            else
            {
                for (int i = 0; i < num_operations; ++i)
                {
                    stream.Get(new StrKey("" + random.Next(0,num_operations - 1)));
                }
                stream.Close();
            }

            // Dump the instrumentation logs
            stream.DumpLogs(exp_directory + "/log");

            // Collect costs usage
            List<string> costs = new List<string>();
            if (doCosts)
            {
                costs.Add(DateTime.UtcNow.Ticks + ": CPU: " + costhelper.getCurrentCpuUsage());
                costs.Add(DateTime.UtcNow.Ticks + ": Network: " + costhelper.getNetworkUsage());
                costs.Add(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated:true)/1000.0f);
                costs.Add(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(this.exp_directory, dataRelated:false)/1000.0f);
                costs.Add(DateTime.UtcNow.Ticks + ": Baseline Storage: " + baselineStorageKV/1000.0f);
            }
            File.AppendAllLines(exp_directory + "/log", costs);

            // sf.deleteStream(streamid, ci);
        }
Ejemplo n.º 42
0
 public override string ToString()
 {
     return(StrKey.EncodeStellarAccountId(Data));
 }
Ejemplo n.º 43
0
 private void AddPicDataStream(string key, byte[] imageBytes)
 {
     StrKey strKey = new StrKey(key);
     ByteValue byteVal = new ByteValue(imageBytes);
     try
     {
         picStream = base.CreateDirStream<StrKey, ByteValue>("H2OAlertsPics", true);
         picStream.Append(strKey, byteVal);
        // logger.Log("WaterAlert picture has been written to {0}.", picStream.Get(strKey).ToString());
         picStream.Close();
     }
     catch (Exception e)
     {
         logger.Log("Error while writing images to dir stream: {0}", e.ToString());
     }
 }
Ejemplo n.º 44
0
        public void IsNotValidEd25519PublicKey(string address)
        {
            var result = StrKey.IsValidEd25519PublicKey(address);

            Assert.IsFalse(result);
        }
Ejemplo n.º 45
0
 public void WriteToStream()
 {
     StrKey key = new StrKey("DummyKey");
     datastream.Append(key, new StrValue("DummyVal"));
     logger.Log("Writing {0} to stream " , datastream.Get(key).ToString());
 }
Ejemplo n.º 46
0
        public static Logger doDiskRaw(StreamOperation op,
                                       int num_operations, int val_size,
                                       StreamFactory.StreamDataType ptype,
                                       string exp_dir)
        {
            Logger logger = new Logger();

            if (op == StreamOperation.RandomKeyRandomValueAppend ||
                op == StreamOperation.SameKeyRandomValueAppend ||
                op == StreamOperation.RandomKeySameValueAppend ||
                op == StreamOperation.SameKeySameValueAppend)
            {
                // Populate the keys and the values
                Random   random = new Random(DateTime.Now.Millisecond);
                StrKey[] keys   = new StrKey[num_operations];
                for (int i = 0; i < num_operations; ++i)
                {
                    keys[i] = new StrKey("" + i);
                }

                Byte[] singleval = new Byte[val_size];
                random.NextBytes(singleval);
                Byte[][] tmp = new Byte[num_operations][];
                if (!(op == StreamOperation.RandomKeySameValueAppend ||
                      op == StreamOperation.SameKeySameValueAppend))
                {
                    for (int i = 0; i < num_operations; ++i)
                    {
                        tmp[i] = new Byte[val_size];
                        random.NextBytes(tmp[i]);
                    }
                }

                StrKey key = new StrKey("ExpKey");
                if (ptype == StreamFactory.StreamDataType.Values)
                {
                    // ValueDataStream type
                    logger.Log("Start Stream Open");
                    FileStream fout = new FileStream(exp_dir + "/DiskRaw", FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite);
                    fout.Seek(0, SeekOrigin.End);
                    BinaryWriter fs_bw = new BinaryWriter(fout);
                    logger.Log("End Stream Open");
                    // TimeSpan span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0));
                    // Console.WriteLine("Time" + span.TotalSeconds);
                    logger.Log("Start Stream Append");
                    for (int i = 0; i < num_operations; ++i)
                    {
                        fs_bw.BaseStream.Seek(0, SeekOrigin.End);

                        /*
                         * if (op == StreamOperation.SameKeySameValueAppend || op == StreamOperation.SameKeyRandomValueAppend)
                         * {
                         *  fs_bw.Write(key.key);
                         * }
                         * else
                         * {
                         */
                        fs_bw.Write(keys[i].key);

                        /*
                         * }
                         */
                        fs_bw.Write(StreamFactory.NowUtc());

                        /*
                         * if (!(op == StreamOperation.RandomKeySameValueAppend ||
                         *  op == StreamOperation.SameKeySameValueAppend))
                         *  fs_bw.Write(tmp[i]);
                         * else
                         */
                        fs_bw.Write(singleval);
                        fs_bw.Flush();
                    }
                    fout.Flush(true);
                    fs_bw.Close();
                    // span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0));
                    // Console.WriteLine("Time" + span.TotalSeconds);
                    logger.Log("End Stream Append");
                }
                else
                {
                    string[] filenames = new string[num_operations];
                    for (int i = 0; i < num_operations; ++i)
                    {
                        filenames[i] = exp_dir + "/DiskRaw" + i;
                    }
                    // FileDataStream type
                    logger.Log("Start Stream Append");
                    for (int i = 0; i < num_operations; ++i)
                    {
                        FileStream fout = new FileStream(filenames[i], FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite);
                        // fout.Seek(0, SeekOrigin.End);
                        BinaryWriter fs_bw = new BinaryWriter(fout);
                        // fs_bw.BaseStream.Seek(0, SeekOrigin.End);

                        /*
                         * if (op == StreamOperation.SameKeySameValueAppend || op == StreamOperation.SameKeyRandomValueAppend)
                         * {
                         *  fs_bw.Write(key.key);
                         * }
                         * else
                         * {
                         */
                        fs_bw.Write(keys[i].key);
                        // }
                        fs_bw.Write(StreamFactory.NowUtc());

                        /*
                         * if (!(op == StreamOperation.RandomKeySameValueAppend ||
                         *  op == StreamOperation.SameKeySameValueAppend))
                         *  fs_bw.Write(tmp[i]);
                         * else
                         */
                        fs_bw.Write(singleval);
                        fout.Flush(true);
                        fs_bw.Close();
                    }
                    logger.Log("End Stream Append");
                }
            }
            else
            {
                // Populate the keys and the values
                Random random = new Random();
                Byte[] val    = new Byte[val_size];

                if (ptype == StreamFactory.StreamDataType.Values)
                {
                    long[] offsets = new long[num_operations];
                    for (int i = 0; i < num_operations; ++i)
                    {
                        offsets[i] = random.Next(0, num_operations - 1) * val_size;
                    }
                    // ValueDataStream type
                    logger.Log("Start Stream Open");
                    FileStream   fin   = new FileStream(exp_dir + "/DiskRaw", FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
                    BinaryReader fs_br = new BinaryReader(fin);
                    logger.Log("End Stream Open");
                    logger.Log("Start Stream Get");
                    logger.Log("Start Stream GetAll");
                    for (int i = 0; i < num_operations; ++i)
                    {
                        fs_br.BaseStream.Seek(offsets[i], SeekOrigin.Begin);
                        fs_br.Read(val, 0, val_size);
                    }
                    fs_br.Close();
                    logger.Log("End Stream Get");
                    logger.Log("End Stream GetAll");
                }
                else
                {
                    string[] filenames = new string[num_operations];
                    for (int i = 0; i < num_operations; ++i)
                    {
                        filenames[i] = exp_dir + "/DiskRaw" + random.Next(0, num_operations);
                    }
                    // FileDataStream type
                    logger.Log("Start Stream Get");
                    logger.Log("Start Stream GetAll");
                    for (int i = 0; i < num_operations; ++i)
                    {
                        FileStream   fin   = new FileStream(filenames[i], FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
                        BinaryReader fs_br = new BinaryReader(fin);
                        fs_br.Read(val, 0, val_size);
                        fs_br.Close();
                    }
                    logger.Log("End Stream Get");
                    logger.Log("End Stream GetAll");
                }
            }
            CostsHelper costhelper = new CostsHelper();

            logger.Log(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(exp_dir, dataRelated: true) / 1000.0f);
            logger.Log(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(exp_dir, dataRelated: false) / 1000.0f);
            return(logger);
        }