A binomial ladder filter maps elements keys to H indexes in an n bit table of bits, which are initially set at random with a probability 0.5. Each index represents a rung on the element's ladder. The indexes set to 1/true are rungs that are below that element on the ladder and the indexes that are 0/false are rungs above the element, or which the element has yet to climb. On avarege, an element that has not been seen before will map to, on average, H/2 index that are set to 1 (true) and H/2 indexes that are set to 0 (false). This means that half the rungs are below the element and half are above it, and so the element is half way up the ladder. If a non-empty subset of the indexes that an element maps to contains 0, observing that element (the Step() method) will cause a random member of that subset to be changed from 0 to 1, causing the element to move one rung up the ladder. To ensure that the fraction of bits set to 1 stays constant, the step operation willsimultaneously clear a random bit selected from subset of indexes in the entire sketch that have value 1. The more times an element has been observed, the higher the expected number of indexes that will have been set, the higher it moves up the ladder until reaching the top (when all of its bits are set). To count the subset of a keys indexes that have been set, one can call the GetRungsBelow method. Natural variance will cause some never-before-seen keys to have more than k/2 bits set and keys that have been observed very rarely (or only in the distant past) to have fewer than k/2 bit set. an element that is observed only a few times (with ~ k/2 + epsilon bits set) will be among ~ .5^{epsilon} false-positive keys that have as many bits set by chance. Thus, until an element has been observed a more substantial number of times it is hard to differentiate from false positives. To have confidence greater than one in a million that an element has been observed, it will take an average of 20 observations.
Inheritance: FilterArray, IBinomialLadderFilter
コード例 #1
0
        /// <summary>
        /// This method will prime the simulator with known-popular passwords so that they are treated
        /// as if the simulator had already observed them (or been configured with them)
        /// </summary>
        /// <returns></returns>
        public void PrimeWithKnownPasswordsAsync(BinomialLadderFilter freqFilter, int numberOfTimesToPrime)
        {

            for (int i = 0; i < numberOfTimesToPrime; i++)
            {
                Parallel.ForEach(_passwordsAlreadyKnownToBePopular,
                    (password) => freqFilter.Step(password));
            }
        }
コード例 #2
0
        public void TwentyObservations()
        {
            BinomialLadderFilter freqFilter = new BinomialLadderFilter(1024*1024*1024, 64);
            string somethingToObserve = "Gosh.  It's a nice day out, isn't it?";

            int observationCount = freqFilter.GetHeight(somethingToObserve);

            for (int i = 0; i < 25; i++)
            {
                int lastCount = freqFilter.Step(somethingToObserve);
                Assert.Equal(observationCount, lastCount);
                observationCount++;
            }

            Assert.Equal(observationCount, freqFilter.GetHeight(somethingToObserve));
        }
コード例 #3
0
        public void Init()
        {
            var builder = new ConfigurationBuilder()
                .SetBasePath("")
                .AddJsonFile("appsettings.json");

            builder.AddEnvironmentVariables();
            Configuration = builder.Build();

            sqlConnectionString = Configuration["Data:ConnectionString"];
            string cloudStorageConnectionString = Configuration["Data:StorageConnectionString"];

            _options = new BlockingAlgorithmOptions();

            DbContextOptionsBuilder<DbUserAccountContext> dbOptionsBuilder = new DbContextOptionsBuilder<DbUserAccountContext>();
            dbOptionsBuilder.UseSqlServer(sqlConnectionString);
            dbOptions = dbOptionsBuilder.Options;

            _CloudStorageAccount = CloudStorageAccount.Parse(cloudStorageConnectionString);
            userAccountControllerFactory = new DbUserAccountControllerFactory(_CloudStorageAccount, dbOptions);
            userAccountController = userAccountControllerFactory.CreateDbUserAccountController();

            RemoteHost localHost = new RemoteHost { Uri = new Uri("http://localhost:35358") };
            MaxWeightHashing<RemoteHost> hosts = new MaxWeightHashing<RemoteHost>(Configuration["Data:UniqueConfigurationSecretPhrase"]);

            LoginAttemptClient<DbUserAccount> loginAttemptClient = new LoginAttemptClient<DbUserAccount>(hosts, localHost);

            _UserAccountRepositoryFactory = new DbUserAccountRepositoryFactory(dbOptions);

            BinomialLadderFilter localPasswordBinomialLadderFilter = new BinomialLadderFilter(
                _options.NumberOfBitsInBinomialLadderFilter_N, _options.HeightOfBinomialLadder_H);

            _loginAttemptController = new LoginAttemptController<DbUserAccount>(
                userAccountControllerFactory, _UserAccountRepositoryFactory,
                localPasswordBinomialLadderFilter,
                new MemoryUsageLimiter(), _options);

            //services.AddEntityFramework()
            //    .AddSqlServer()
            //    .AddDbContext<DbUserAccountContext>(opt => opt.UseSqlServer(sqlConnectionString));
            //DbUserAccountContext context = new DbUserAccountContext();

            //var db = new DbContextOptionsBuilder();
            //db.UseInMemoryDatabase();
            //_context = new MyContext(db.Options);

        }
コード例 #4
0
        public static TestConfiguration InitTest(BlockingAlgorithmOptions options = default(BlockingAlgorithmOptions))
        {
            if (options == null)
                options = new BlockingAlgorithmOptions();

            TestConfiguration configuration = new TestConfiguration();
            configuration.MyBlockingAlgorithmOptions = options ?? new BlockingAlgorithmOptions();

            //configuration.MyResponsibleHosts = new MaxWeightHashing<RemoteHost>("FIXME-uniquekeyfromconfig");
            //RemoteHost localHost = new RemoteHost { Uri = new Uri("http://localhost:80") };
            //configuration.MyResponsibleHosts.Add("localhost", localHost);
            //IStableStore stableStore = configuration.StableStore = new MemoryOnlyStableStore();
            
            //configuration.MyLoginAttemptClient = new LoginAttemptClient(configuration.MyResponsibleHosts, localHost);

            MemoryUsageLimiter memoryUsageLimiter = new MemoryUsageLimiter();

            BinomialLadderFilter localPasswordBinomialLadderFilter =
            new BinomialLadderFilter(options.NumberOfBitsInBinomialLadderFilter_N, options.HeightOfBinomialLadder_H);

            //MultiperiodFrequencyTracker<string> localPasswordFrequencyTracker =
            //        new MultiperiodFrequencyTracker<string>(
            //            options.NumberOfPopularityMeasurementPeriods,
            //            options.LengthOfShortestPopularityMeasurementPeriod,
            //            options.FactorOfGrowthBetweenPopularityMeasurementPeriods);
            
            configuration.MyAccountFactory = new MemoryOnlyUserAccountFactory();
            configuration.MemUserAccountController = new MemoryUserAccountController();

            LoginAttemptController<MemoryUserAccount> myLoginAttemptController = 
                new LoginAttemptController<MemoryUserAccount>(
                new MemoryUserAccountControllerFactory(),
                configuration.MyAccountFactory,
                localPasswordBinomialLadderFilter,
                memoryUsageLimiter, configuration.MyBlockingAlgorithmOptions);

            configuration.MyLoginAttemptClient = myLoginAttemptController;

            //configuration.MyLoginAttemptClient.SetLocalLoginAttemptController(myLoginAttemptController);
            return configuration;
        }
コード例 #5
0
ファイル: Startup.cs プロジェクト: Microsoft/StopGuessing
        // This method gets called by the runtime. Use this method to add services to the container.
        public void ConfigureServices(IServiceCollection services)
        {
            // Add framework services.
            services.AddMvc();

            string sqlConnectionString = Configuration["Data:ConnectionString"];
            services.AddDbContext<DbUserAccountContext>(
                opt => opt.UseSqlServer(sqlConnectionString));

            // Uncomment the following line to add Web API services which makes it easier to port Web API 2 controllers.
            // You will also need to add the Microsoft.AspNet.Mvc.WebApiCompatShim package to the 'dependencies' section of project.json.
            // services.AddWebApiConventions();

            BlockingAlgorithmOptions options = new BlockingAlgorithmOptions();

            services.AddSingleton<BlockingAlgorithmOptions>(x => options);

            RemoteHost localHost = new RemoteHost { Uri = new Uri("http://localhost:35358") };
            services.AddSingleton<RemoteHost>(x => localHost);

            MaxWeightHashing<RemoteHost> hosts = new MaxWeightHashing<RemoteHost>(Configuration["Data:UniqueConfigurationSecretPhrase"]);
            hosts.Add("localhost", localHost);
            services.AddSingleton<IDistributedResponsibilitySet<RemoteHost>>(x => hosts);

            string cloudStorageConnectionString = Configuration["Data:StorageConnectionString"];
            CloudStorageAccount cloudStorageAccount = CloudStorageAccount.Parse(cloudStorageConnectionString);
            services.AddSingleton<CloudStorageAccount>(a => cloudStorageAccount);

            services.AddSingleton<MemoryUsageLimiter, MemoryUsageLimiter>();

            if (hosts.Count > 0)
            {
                DistributedBinomialLadderFilterClient dblfClient = new DistributedBinomialLadderFilterClient(
                    options.NumberOfVirtualNodesForDistributedBinomialLadder,
                    options.HeightOfBinomialLadder_H,
                    hosts,
                    options.PrivateConfigurationKey,
                    options.MinimumBinomialLadderFilterCacheFreshness);
                // If running as a distributed system
                services.AddSingleton<IBinomialLadderFilter, DistributedBinomialLadderFilterClient>(x => dblfClient);

                DistributedBinomialLadderFilterController filterController =
                    new DistributedBinomialLadderFilterController(dblfClient, options.NumberOfBitsPerShardInBinomialLadderFilter, options.PrivateConfigurationKey);
                services.AddSingleton<DistributedBinomialLadderFilterController>(x => filterController);

                //services.AddSingleton<IFrequenciesProvider<string>>(x =>
                //    new IncorrectPasswordFrequencyClient(hosts, options.NumberOfRedundantHostsToCachePasswordPopularity));
            }
            else
            {
                BinomialLadderFilter localPasswordBinomialLadderFilter =
                    new BinomialLadderFilter(options.NumberOfBitsInBinomialLadderFilter_N, options.HeightOfBinomialLadder_H);
                services.AddSingleton<IBinomialLadderFilter>(x => localPasswordBinomialLadderFilter);
            }

            LoginAttemptClient<DbUserAccount> loginAttemptClient = new LoginAttemptClient<DbUserAccount>(hosts, localHost);
            services.AddSingleton<IUserAccountRepositoryFactory<DbUserAccount>, DbUserAccountRepositoryFactory>();
            services.AddSingleton<IUserAccountControllerFactory<DbUserAccount>, DbUserAccountControllerFactory>();
            //LoginAttemptController<DbUserAccount> loginAttemptController = new LoginAttemptController<DbUserAccount>( 
            //     new DbUserAccountControllerFactory(cloudStorageAccount), new DbUserAccountRepositoryFactory());
            services.AddSingleton<ILoginAttemptClient, LoginAttemptClient<DbUserAccount>>(i => loginAttemptClient);
            services.AddSingleton<ILoginAttemptController, LoginAttemptController<DbUserAccount>>();
        }
コード例 #6
0
ファイル: Simulator.cs プロジェクト: Microsoft/StopGuessing
        //public void ReduceMemoryUsage(object sender, MemoryUsageLimiter.ReduceMemoryUsageEventParameters parameters)
        //{
            //_ipHistoryCache.RecoverSpace(parameters.FractionOfMemoryToTryToRemove);
        //}

        public Simulator(DebugLogger logger, string path, ExperimentalConfiguration myExperimentalConfiguration, SimulatedPasswords simPasswords)
        {
            
            _simPasswords = simPasswords;
            _logger = logger;
            _AttackAttemptsWithValidPasswords = //System.IO.TextWriter.Synchronized 
                new ConcurrentStreamWriter(path + "AttackAttemptsWithValidPasswords.txt");
                //(new StreamWriter(new FileStream(path + "AttackAttemptsWithValidPasswords.txt", FileMode.CreateNew, FileAccess.Write)));
            _LegitimateAttemptsWithValidPasswords = //System.IO.TextWriter.Synchronized
                new ConcurrentStreamWriter(path + "LegitimateAttemptsWithValidPasswords.txt");
            //(new StreamWriter(new FileStream(path + "LegitiamteAttemptsWithValidPasswords.txt", FileMode.CreateNew, FileAccess.Write)));
            _OtherAttempts = //System.IO.TextWriter.Synchronized
                new ConcurrentStreamWriter(path + "OtherAttempts.txt");
                //(new StreamWriter(new FileStream(path + "OtherAttempts.txt", FileMode.CreateNew, FileAccess.Write)));
            _logger.WriteStatus("Entered Simulator constructor");
            _experimentalConfiguration = myExperimentalConfiguration;
            BlockingAlgorithmOptions options = _experimentalConfiguration.BlockingOptions;
            
            _logger.WriteStatus("Creating binomial ladder");
            _binomialLadderFilter =
                new BinomialLadderFilter(options.NumberOfBitsInBinomialLadderFilter_N, options.HeightOfBinomialLadder_H);
            _ipHistoryCache = new ConcurrentDictionary<IPAddress, SimIpHistory>(); // new SelfLoadingCache<IPAddress, SimIpHistory>(address => new SimIpHistory(options.NumberOfFailuresToTrackForGoingBackInTimeToIdentifyTypos));
            _userAccountController = new SimulatedUserAccountController();

            //_memoryUsageLimiter = new MemoryUsageLimiter();
            //_memoryUsageLimiter.OnReduceMemoryUsageEventHandler += ReduceMemoryUsage;

            _recentIncorrectPasswords = new AgingMembershipSketch(16, 128 * 1024);

            _logger.WriteStatus("Exiting Simulator constructor");
        }