Esempio n. 1
0
        private static Stopwatch ParseDisks(Action<Disk> addToBatch)
        {
            int i = 0;
            var parser = new Parser();
            var buffer = new byte[1024 * 1024];// more than big enough for all files

            var sp = Stopwatch.StartNew();

            using (var bz2 = new BZip2InputStream(File.Open(@"D:\Data\freedb-complete-20120101.tar.bz2", FileMode.Open)))
            using (var tar = new TarInputStream(bz2))
            {
                TarEntry entry;
                while ((entry = tar.GetNextEntry()) != null)
                {
                    if (entry.Size == 0 || entry.Name == "README" || entry.Name == "COPYING")
                        continue;
                    var readSoFar = 0;
                    while (true)
                    {
                        var read = tar.Read(buffer, readSoFar, ((int)entry.Size) - readSoFar);
                        if (read == 0)
                            break;

                        readSoFar += read;
                    }
                    // we do it in this fashion to have the stream reader detect the BOM / unicode / other stuff
                    // so we can read the values properly
                    var fileText = new StreamReader(new MemoryStream(buffer, 0, readSoFar)).ReadToEnd();
                    try
                    {
                        var disk = parser.Parse(fileText);
                        addToBatch(disk);
                        if (i++ % 1000 == 0)
                            Console.Write("\r{0} {1:#,#}  {2}         ", entry.Name, i, sp.Elapsed);

                        if (i % 50000 == 0)
                            return sp;
                    }
                    catch (Exception e)
                    {
                        Console.WriteLine();
                        Console.WriteLine(entry.Name);
                        Console.WriteLine(e);
                        return sp;
                    }
                }
            }
            return sp;
        }
Esempio n. 2
0
        private void ParseEntries()
        {
            var parser = new Parser();

            while (true)
            {
                var entry = _entries.Take();
                if (entry == null)
                    break;
                var disk = parser.Parse(entry);
                _disks.Add(disk);
                Interlocked.Increment(ref parsed);
            }

            _disks.Add(null);
        }
Esempio n. 3
0
        private static void ParseDisks(BulkInsertOperation insert)
        {
            int i = 0;
            var parser = new Parser();
            var buffer = new byte[1024*1024];// more than big enough for all files

            using (var bz2 = new BZip2InputStream(File.Open(@"D:\Scratch\freedb-complete-20150101.tar.bz2", FileMode.Open)))
            using (var tar = new TarInputStream(bz2))
            {
                TarEntry entry;
                while((entry=tar.GetNextEntry()) != null)
                {
                    if(entry.Size == 0 || entry.Name == "README" || entry.Name == "COPYING")
                        continue;

                    var readSoFar = 0;
                    while(true)
                    {
                        var read = tar.Read(buffer, readSoFar, ((int) entry.Size) - readSoFar);
                        if (read == 0)
                            break;

                        readSoFar += read;
                    }
                    // we do it in this fashion to have the stream reader detect the BOM / unicode / other stuff
                    // so we can read the values properly
                    var fileText = new StreamReader(new MemoryStream(buffer,0, readSoFar)).ReadToEnd();
                    try
                    {
                        var disk = parser.Parse(fileText);
                        insert.Store(disk);
                    }
                    catch (Exception e)
                    {
                        Console.WriteLine();
                        Console.WriteLine(entry.Name);
                        Console.WriteLine(e);
                    }
                }
            }
        }