public static Chain ReadRawData(BinaryReader br, MarkovWordGenerator gen) { Chain ch = new Chain(gen); ch.m_totalSize = br.ReadInt32(); int size = br.ReadInt32(); for (int i = 0; i < size; i++) { ch.m_letters.Add(br.ReadChar(), br.ReadInt32()); } return(ch); }
private static void ProcessFile( List<char> charSet, FileInfo inFile ) { MarkovWordGenerator gen = new MarkovWordGenerator( 3 ); Console.WriteLine( "Processing file: {0}", inFile.Name ); int counter = 0; List<string> skippedWords = new List<string>(); using ( StreamReader sr = new StreamReader( inFile.FullName, Encoding.UTF8 ) ) { bool skipWord; while ( !sr.EndOfStream ) { ++counter; if ( counter%50000 == 0 ) Console.WriteLine( counter ); skipWord = false; string line = sr.ReadLine(); for ( int i = 0; i < line.Length; i++ ) { char c = line[i]; if ( !charSet.Contains( c ) ) { skippedWords.Add( line ); skipWord = true; } } if ( !skipWord ) gen.SampleWord( line ); } } FileInfo outFile = new FileInfo( Path.Combine( "output", Path.ChangeExtension( inFile.Name, ".bin" ) ) ); using ( FileStream fs = outFile.Open( FileMode.Create, FileAccess.Write, FileShare.None ) ) using ( BinaryWriter bw = new BinaryWriter( fs ) ) gen.DumpRawData( bw ); outFile = new FileInfo( Path.Combine( "output", inFile.Name + " - Skipped.txt" ) ); using( FileStream fs = outFile.Open( FileMode.Create, FileAccess.Write, FileShare.None ) ) using( StreamWriter sw = new StreamWriter( fs, Encoding.UTF8 ) ) foreach ( string skippedWord in skippedWords ) sw.WriteLine( skippedWord ); }
private bool LoadMarkovChains() { DirectoryInfo dir = new DirectoryInfo( "namedata" ); if( !dir.Exists ) return false; FileInfo[] files = dir.GetFiles( "*.bin" ); var groups = files.GroupBy( f => f.Name[0] ); MarkovWordGenerator gen; List<MarkovWordGenerator> genList; foreach( var gr in groups ) { genList = new List<MarkovWordGenerator>(); foreach( FileInfo file in gr ) { using( FileStream fs = new FileStream( file.FullName, FileMode.Open, FileAccess.Read, FileShare.Read ) ) using( BinaryReader br = new BinaryReader( fs ) ) { gen = new MarkovWordGenerator( br ); genList.Add( gen ); } } m_wordGenerators.Add( genList ); } if ( m_wordGenerators.Count == 0 ) return false; return true; }
public Chain(MarkovWordGenerator gen) { m_letters = new Dictionary <char, int>(); m_totalSize = 0; m_parentGen = gen; }
public static Chain ReadRawData( BinaryReader br, MarkovWordGenerator gen ) { Chain ch = new Chain( gen ); ch.m_totalSize = br.ReadInt32(); int size = br.ReadInt32(); for( int i = 0; i < size; i++ ) ch.m_letters.Add( br.ReadChar(), br.ReadInt32() ); return ch; }
public Chain( MarkovWordGenerator gen ) { m_letters = new Dictionary<char, int>(); m_totalSize = 0; m_parentGen = gen; }