/// <summary>
    /// Make sure we find exactly the known data we're after
    /// </summary>
    static void CanLocateKnownData(TestContext context)
    {
        var index = new Dictionary<string, byte[]>();
        var encoder = new Base128Encoder();

        // The term "cat" matches documents 2,4,6,8
        var docsWithCat = new List<uint>() { 2, 4, 6, 8 };
        index["cat"] = encoder.EncodeList(docsWithCat);

        // The term "dog" matches documents 6,8,10,12
        var docsWithDog = new List<uint>() { 6, 8, 10, 12 };
        index["dog"] = encoder.EncodeList(docsWithDog);

        // query for docs that have both "cat" and "dog"
        var results = new HashSet<uint>();
        var firstPass = true;

        foreach (var pair in index)
        {
            var idList = encoder.DecodeList(pair.Value);
            if (firstPass)
            {
                firstPass = false;
                results.UnionWith(idList);
            }
            else
            {
                results.IntersectWith(idList);
            }
        }

        // Should have only found 6 and 8
        Assert.AreEqual(2, results.Count);
        Assert.AreEqual((uint)6, results.ElementAt(0));
        Assert.AreEqual((uint)8, results.ElementAt(1));
    }
    /// <summary>
    /// Helper method to add index entries
    /// </summary>
    static void addIndexValue(string termName, 
        string termValue, 
        uint docId, 
        uint position, 
        Dictionary<uint, TermValuePair> termHashMap, 
        Dictionary<uint, DocPositionPair> docHashMap, 
        Dictionary<uint, byte[]> index, 
        Base128Encoder encoder)
    {
        var termPair = new TermValuePair(termName, termValue);
        var termPairHash = termPair.ComputeHashValue();

        if (!termHashMap.ContainsKey(termPairHash))
        {
            termHashMap[termPairHash] = termPair;
        }

        var docPair = new DocPositionPair(docId, position);
        var docPairHash = docPair.ComputeHashValue();

        if (!docHashMap.ContainsKey(docPairHash))
        {
            docHashMap[docPairHash] = docPair;
        }

        if (!index.ContainsKey(termPairHash))
        {
            index[termPairHash] = encoder.EncodeList(new List<uint>() { docPairHash });
        }
        else
        {
            var currentList = encoder.DecodeList(index[termPairHash]);
            currentList.Add(docPairHash);
            index[termPairHash] = encoder.EncodeList(currentList);
        }
    }
    /// <summary>
    /// Tests the varint decoding performance.
    /// </summary>
    static void LoadTestVarintDecoding(TestContext context)
    {
        // Test config
        var TERMS = 50000;
        var DOCS_PER_TERM = 10000;
        var TOTAL_DOCS = 100000;

        // test support
        var sw = new Stopwatch();
        var rng = new Random();

        // Our simple inverted index and an encoder
        // Note the index does not map term to list-of-docid...
        // Instead, it maps term to compressed range.
        var index = new Dictionary<string, byte[]>();
        var encoder = new Base128Encoder();

        ////////////////////////////////////////////////////////
        //
        // Create index data
        //

        for (var j = 0; j < TERMS; j++)
        {
            var docLinks = new List<uint>();

            // add document links
            for (var m = 0; m < DOCS_PER_TERM; m++)
            {
                docLinks.Add((uint)rng.Next(1, TOTAL_DOCS));
            }

            // compress the docID list
            // We'll just use 'j' in string form as our term
            index[j.ToString()] = encoder.EncodeList(docLinks);
        }

        ////////////////////////////////////////////////////////
        //
        // Search index data
        //

        var decodeCount = 0;
        sw.Start();

        // how about just a simple linear scan for now
        foreach (var termPair in index)
        {
            var decodedList = encoder.DecodeList(termPair.Value);
            decodeCount += decodedList.Count;
        }

        ////////////////////////////////////////////////////////
        //
        // Results summary
        //

        sw.Stop();
        context.WriteLine(string.Format("Decoded {0} keys in {1} ms", decodeCount, sw.ElapsedMilliseconds));
    }
    /// <summary>
    /// Tests a decode / set-operation scenario
    /// </summary>
    static void LoadTestScenario(TestContext context)
    {
        // Test config
        var TERMS = 50000;
        var DOCS_PER_TERM = 10000;
        var TOTAL_DOCS = 100000;

        // test support
        var sw = new Stopwatch();
        var rng = new Random();

        // Our simple inverted index and an encoder
        // Note the index does not map term to list-of-docid...
        // Instead, it maps term to compressed range.
        var index = new Dictionary<string, byte[]>();
        var encoder = new Base128Encoder();

        ////////////////////////////////////////////////////////
        //
        // Create index data
        //

        for (var j = 0; j < TERMS; j++)
        {
            var docLinks = new List<uint>();

            // add document links
            for (var m = 0; m < DOCS_PER_TERM; m++)
            {
                docLinks.Add((uint)rng.Next(1, TOTAL_DOCS));
            }

            // compress the docID list
            // We'll just use 'j' in string form as our term
            index[j.ToString()] = encoder.EncodeList(docLinks);
        }

        ////////////////////////////////////////////////////////
        //
        // Search index data
        //

        // Let's search for 'terms' that begin with '5000'
        // We then want all docs that are in both terms
        var query = new Predicate<string>((s) => s.StartsWith("5000"));
        var outSet = new HashSet<uint>();
        var firstPass = true;
        sw.Start();

        // how about just a simple linear scan for now
        foreach (var termPair in index)
        {
            if (query(termPair.Key))
            {
                // decode
                var decodedList = encoder.DecodeList(termPair.Value);

                // can't intersect with an empty set - always result in nullset.
                if (firstPass)
                {
                    firstPass = false;
                    outSet.UnionWith(decodedList);
                }
                else
                {
                    outSet.IntersectWith(decodedList);
                }
            }
        }

        ////////////////////////////////////////////////////////
        //
        // Results summary
        //

        sw.Stop();
        context.WriteLine(string.Format("Found {0} results", outSet.Count));
        context.WriteLine(string.Format("Query time: {0}ms", sw.ElapsedMilliseconds));
    }
    /// <summary>
    /// Make sure we can get a list back in the same forms.
    /// </summary>
    static void ListEncodesAndDecodes(TestContext context)
    {
        var rawList = new List<uint>();

        for (var j = 0; j < 100; j++)
        {
            rawList.Add((uint)j);
        }

        var encoder = new Base128Encoder();
        var decodedList = encoder.DecodeList(encoder.EncodeList(rawList));

        rawList.Sort();
        decodedList.Sort();

        Assert.AreEqual(rawList.Count, decodedList.Count);

        for (var m = 0; m < rawList.Count; m++)
        {
            Assert.AreEqual(rawList[m], decodedList[m]);
        }
    }
    /// <summary>
    /// Just a check to see how much space we're saving
    /// </summary>
    static void CompressionYieldsLessStorage(TestContext context)
    {
        // create a list of 2,000,000 uints
        // at 4bytes per uint, this should be ~7.8 - 8.0 MB
        var TEST_SIZE = 2000000;
        var docIds = new List<UInt32>(TEST_SIZE);
        var rng = new Random();

        for (var j = 0; j < TEST_SIZE; j++)
        {
            docIds.Add((uint)rng.Next(1, Int32.MaxValue));
        }

        // now compress that list
        var encoder = new Base128Encoder();
        var packedData = encoder.EncodeList(docIds);

        // How much space did we save
        context.WriteLine(String.Format("Raw size: {0} bytes", TEST_SIZE * 4));
        context.WriteLine(String.Format("Packed size: {0} bytes", packedData.Length));
    }