Пример #1
0
        public void BatchSaveTest()
        {
            Dictionary <int, byte[]> data = new Dictionary <int, byte[]>();
            Dictionary <int, IPtr>   ptrs = new Dictionary <int, IPtr>();

            using (storage = CreateNewStorage(true))
            {
                data[1] = CreateData(1, 100);
                data[2] = CreateData(2, 200);
                data[3] = CreateData(3, 300);
                ptrs[1] = storage.Save(data[1]);
                ptrs[2] = storage.Save(data[2]);
                var p = storage.BatchSave(new Tuple <IPtr, byte[]>[]
                {
                    new Tuple <IPtr, byte[]>(ptrs[1], data[1]),
                    new Tuple <IPtr, byte[]>(ptrs[2], data[2]),
                    new Tuple <IPtr, byte[]>(null, data[3]),
                });
                for (int i = 0; i < p.Length; ++i)
                {
                    ptrs[i + 1] = p[i];
                }
                foreach (var key in data.Keys)
                {
                    Microsoft.VisualStudio.TestTools.UnitTesting.Assert.IsTrue(
                        data[key].SequenceEqual(storage.Load(ptrs[key])));
                }
            }
        }
Пример #2
0
        static void Main(string[] args)
        {
            var fsStorage = new FsStorage();
            ArtistNameExporter artistNameExporter = new ArtistNameExporter(fsStorage);

            using (var fs = File.CreateText("artistNames.csv"))
            {
                fs.WriteLine("Id,Artist,Role");
                int i = 1;
                foreach (var artistWithRole in artistNameExporter.GetFullListOfArtistNames(@"c:\users\octo\Music\",
                                                                                           new[] { ".mp3", ".flac", ".wma", ".m4a" }))
                {
                    Console.WriteLine("{2}. Writing: {0} as a {1}.", artistWithRole.Name, artistWithRole.Role, i);
                    fs.WriteLine($"art{i++},{artistWithRole.Name},{artistWithRole.Role}");
                }
            }
        }
Пример #3
0
        private FsIndexedStorage <int> CreateNewStorageWithLogging(bool clear)
        {
            if (clear)
            {
                operations = new List <string>();
                Clear();
            }
            var db    = new FsStorage(dir, prefix, maxDbFileLength: 100 * 1024);
            var index = new FsIndex <int>(dir, prefix, FsStorage.CreatePtr, BitConverter.GetBytes, bytes => BitConverter.ToInt32(bytes, 0));

#if DEBUG
            db.onReading += (ptr, bytes) => { Thread.Sleep(rnd.Next(200)); lock (operations) operations.Add(String.Format("[{0}] read from ptr {1} value {2}", Thread.CurrentThread.ManagedThreadId, ptr.ToString(), BitConverter.ToInt32(bytes, 0))); };
            db.onWriting += (ptr, bytes) => { Thread.Sleep(rnd.Next(200)); lock (operations) operations.Add(String.Format("[{0}] write to ptr {1} value {2}", Thread.CurrentThread.ManagedThreadId, ptr.ToString(), BitConverter.ToInt32(bytes, 0))); };

            index.onGet += (key, ptr) => { Thread.Sleep(rnd.Next(200)); lock (operations) operations.Add(String.Format("[{0}] get by key {1} ptr {2}", Thread.CurrentThread.ManagedThreadId, key, ptr != null ? ptr.ToString() : "")); };
            index.onSet += (key, ptr) => { Thread.Sleep(rnd.Next(200)); lock (operations) operations.Add(String.Format("[{0}] set by key {1} ptr {2}", Thread.CurrentThread.ManagedThreadId, key, ptr.ToString())); };
            index.onRem += (key, ptr) => { Thread.Sleep(rnd.Next(200)); lock (operations) operations.Add(String.Format("[{0}] rem key {1} with ptr {2}", Thread.CurrentThread.ManagedThreadId, key, ptr.ToString())); };
#endif
            return(new FsIndexedStorage <int>(db, index));
        }
Пример #4
0
        static void Main(string[] args)
        {
            var result = Parser.Default.ParseArguments <Options>(args);

            if (result.Tag == ParserResultType.Parsed)
            {
                var options = ((Parsed <Options>)result).Value;
                Console.WriteLine("Executing with argument: >>>{0}<<<", options.RootPath);
                var configSection = ((Tags2FilesConfigSection)ConfigurationManager.GetSection("tagsToFiles"));
                var extensions    =
                    configSection.Extensions
                    .Cast <ExtensionElement>().Select(e => e.Value).ToArray();
                var individualRule = configSection.Rules
                                     .Cast <RuleElement>().FirstOrDefault(r => r.Type == RuleType.Individual);
                if (individualRule == null)
                {
                    throw new ConfigurationErrorsException("no individual rule.");
                }
                var collectionRule = configSection.Rules
                                     .Cast <RuleElement>().FirstOrDefault(r => r.Type == RuleType.Collection);
                if (collectionRule == null)
                {
                    throw new ConfigurationErrorsException("no collection rule.");
                }

                IStorage storage = new FsStorage();
                storage.Directory.DirectoryRead  += FileEnumerator_DirectoryRead;
                storage.Directory.ExtensionsRead += FileEnumerator_ExtensionsRead;

                T2FProcessor t2FProcessor = new T2FProcessor(storage,
                                                             new TagFileProcessor());
                t2FProcessor.Start(options.RootPath, extensions, individualRule.Pattern,
                                   collectionRule.Pattern, configSection.ActionType.Type);
                return;
            }
            Console.WriteLine(CommandLine.Text.HelpText.AutoBuild(result));
        }
Пример #5
0
        public void SimpleSaveLoadTest()
        {
            using (storage = CreateNewStorage(true))
            {
                for (int i = 0; i < N; ++i)
                {
                    int key = rnd.Next(100);
                    if (i < 200 || i % 2 == 0)
                    {
                        Store(key, rnd.Next());
                    }
                    else
                    {
                        Read(key);
                    }

                    if (i % 1000 == 0)
                    {
                        storage.Flush();
                    }
                }

                Read(200);
                Store(200, 1240281);
                Read(200);
                Store(200, 291483);
                Read(200);
            }

            using (storage = CreateNewStorage(false))
            {
                for (int i = 0; i < N; ++i)
                {
                    int key = rnd.Next(100);
                    if (i < 200 || i % 2 == 0)
                    {
                        Store(key, rnd.Next());
                    }
                    else
                    {
                        Read(key);
                    }

                    if (i % 1000 == 0)
                    {
                        storage.Flush();
                    }
                }

                // test wrong ptr

                /*
                 * IPtr wrong = new FsStorage.Ptr() {Capacity = 3600, FileNum = 0, Position = 140};
                 * control_index[1001] = wrong;
                 * control_storage[1001] = null;
                 * Read(1001);
                 */

                int big = 1024 * 1024 * 2 / 4;
                Read(big);
                Store(big, rnd.Next());
                Read(big);
            }
        }
Пример #6
0
        public void PerformanceBatchTest()
        {
            int  N, NThreads, NBatch, NKeys;
            bool SAVETEST = true, READTEST = true;

            NKeys = 5000;
            byte[][] data = new byte[NKeys][];
            IPtr[]   ptrs = new IPtr[NKeys];

            N        = 100000;
            NThreads = 10;
            NBatch   = 5000;
            int NReadThreads = 8, NWriteThreads = 2;

            using (storage = CreateNewStorage(true))
            {
                for (int i = 0; i < data.Length; ++i)
                {
                    data[i] = CreateData(i, 100 + rnd.Next(5000));
                    ptrs[i] = storage.Save(data[i]);
                }


                if (SAVETEST)
                {
                    double[]     KbytesSaved = Enumerable.Repeat(0.0, NThreads).ToArray();
                    Action <int> save        = (tid) =>
                    {
                        for (int i = 0; i < N; i += NBatch)
                        {
                            Tuple <IPtr, byte[]>[] values = Enumerable.Repeat(0, NBatch).Select(_ => Tuple.Create(ptrs[rnd.Next(NKeys)], data[rnd.Next(NKeys)])).ToArray();
                            storage.BatchSave(values);
                            KbytesSaved[tid] += values.Sum(v => (long)v.Item2.Length) / 1024.0;
                        }
                    };
                    var tsave = MeasureTimeOfThredas(NThreads, save);
                    Console.WriteLine("" + (N * NThreads) + " save operations in " + NThreads + " threads elapsed: " +
                                      tsave);
                    Console.WriteLine("Save Operations in 1 sec: " + ((double)N * NThreads / tsave.TotalSeconds).ToString("### ###"));
                    Console.WriteLine("KBytes saved in 1 sec: " +
                                      (KbytesSaved.Sum() / tsave.TotalSeconds).ToString("### ### ### Kb"));
                    Console.WriteLine("Total KBytes saved: " + KbytesSaved.Sum());
                }


                if (READTEST)
                {
                    double[]     KbytesReaded = Enumerable.Repeat(0.0, NThreads).ToArray();
                    Action <int> read         = (tid) =>
                    {
                        for (int i = 0; i < N; i += NBatch)
                        {
                            var keys   = rnd.NextN(NKeys, NBatch).Select(key => ptrs[key]).ToArray();
                            var values = storage.BatchLoad(keys);
                            KbytesReaded[tid] += values.Sum(v => v != null ? (long)v.Item2.Length : (long)0) / 1024.0;
                        }
                    };
                    var tread = MeasureTimeOfThredas(NThreads, read);
                    Console.WriteLine("" + (N * NThreads) + " read operations in " + NThreads + " threads elapsed: " +
                                      tread);
                    Console.WriteLine("Read Operations in 1 sec: " + ((double)N * NThreads / tread.TotalSeconds).ToString("### ###"));
                    Console.WriteLine("KBytes readed in 1 sec: " +
                                      (KbytesReaded.Sum() / tread.TotalSeconds).ToString("### ### ##0 Kb"));
                    Console.WriteLine("Total KBytes readed: " + KbytesReaded.Sum());
                }

                if (SAVETEST && READTEST)
                {
                    NThreads = NReadThreads + NWriteThreads;
                    double[]     KbytesProcessed = Enumerable.Repeat(0.0, NThreads).ToArray();
                    Action <int> action          = (tid) =>
                    {
                        for (int i = 0; i < N; i += NBatch)
                        {
                            Tuple <IPtr, byte[]>[] values;
                            if (tid < NWriteThreads)
                            {
                                var keys = rnd.NextN(NKeys, NBatch).Select(key => ptrs[key]).ToArray();
                                values = storage.BatchLoad(keys).ToArray();
                            }
                            else
                            {
                                values = Enumerable.Repeat(0, NBatch).Select(_ => Tuple.Create(ptrs[rnd.Next(NKeys)], data[rnd.Next(NKeys)])).ToArray();
                                storage.BatchSave(values);
                            }
                            KbytesProcessed[tid] += values.Sum(v => v != null ? (long)v.Item2.Length : (long)0) / 1024.0;
                        }
                    };
                    var time = MeasureTimeOfThredas(NThreads, action);
                    Console.WriteLine("" + (N * NReadThreads) + " read operations in " + NReadThreads + " threads and \n" +
                                      "" + (N * NWriteThreads) + " write operations in " + NWriteThreads + " threads elapsed: " +
                                      time);
                    Console.WriteLine("Read|Write Operations in 1 sec: " + ((double)N * NThreads / time.TotalSeconds).ToString("### ###"));
                    Console.WriteLine("KBytes readed|writed in 1 sec: " +
                                      (KbytesProcessed.Sum() / time.TotalSeconds).ToString("### ### ### Kb"));
                    Console.WriteLine("Total KBytes processed: " + KbytesProcessed.Sum());
                }
            }
        }
Пример #7
0
 public TagsProvider(FsStorage fsStorage)
 {
     _fsStorage = fsStorage;
 }