public static Chain ReadRawData(BinaryReader br, MarkovWordGenerator gen)
{
Chain ch = new Chain(gen);
ch.m_totalSize = br.ReadInt32();
int size = br.ReadInt32();
for (int i = 0; i < size; i++)
{
ch.m_letters.Add(br.ReadChar(), br.ReadInt32());
}
return(ch);
}
private static void ProcessFile( List<char> charSet, FileInfo inFile )
{
MarkovWordGenerator gen = new MarkovWordGenerator( 3 );
Console.WriteLine( "Processing file: {0}", inFile.Name );
int counter = 0;
List<string> skippedWords = new List<string>();
using ( StreamReader sr = new StreamReader( inFile.FullName, Encoding.UTF8 ) )
{
bool skipWord;
while ( !sr.EndOfStream )
{
++counter;
if ( counter%50000 == 0 )
Console.WriteLine( counter );
skipWord = false;
string line = sr.ReadLine();
for ( int i = 0; i < line.Length; i++ )
{
char c = line[i];
if ( !charSet.Contains( c ) )
{
skippedWords.Add( line );
skipWord = true;
}
}
if ( !skipWord )
gen.SampleWord( line );
}
}
FileInfo outFile = new FileInfo( Path.Combine( "output", Path.ChangeExtension( inFile.Name, ".bin" ) ) );
using ( FileStream fs = outFile.Open( FileMode.Create, FileAccess.Write, FileShare.None ) )
using ( BinaryWriter bw = new BinaryWriter( fs ) )
gen.DumpRawData( bw );
outFile = new FileInfo( Path.Combine( "output", inFile.Name + " - Skipped.txt" ) );
using( FileStream fs = outFile.Open( FileMode.Create, FileAccess.Write, FileShare.None ) )
using( StreamWriter sw = new StreamWriter( fs, Encoding.UTF8 ) )
foreach ( string skippedWord in skippedWords )
sw.WriteLine( skippedWord );
}
private bool LoadMarkovChains()
{
DirectoryInfo dir = new DirectoryInfo( "namedata" );
if( !dir.Exists )
return false;
FileInfo[] files = dir.GetFiles( "*.bin" );
var groups = files.GroupBy( f => f.Name[0] );
MarkovWordGenerator gen;
List<MarkovWordGenerator> genList;
foreach( var gr in groups )
{
genList = new List<MarkovWordGenerator>();
foreach( FileInfo file in gr )
{
using( FileStream fs = new FileStream( file.FullName, FileMode.Open, FileAccess.Read, FileShare.Read ) )
using( BinaryReader br = new BinaryReader( fs ) )
{
gen = new MarkovWordGenerator( br );
genList.Add( gen );
}
}
m_wordGenerators.Add( genList );
}
if ( m_wordGenerators.Count == 0 )
return false;
return true;
}
public Chain(MarkovWordGenerator gen)
{
m_letters = new Dictionary <char, int>();
m_totalSize = 0;
m_parentGen = gen;
}
public static Chain ReadRawData( BinaryReader br, MarkovWordGenerator gen )
{
Chain ch = new Chain( gen );
ch.m_totalSize = br.ReadInt32();
int size = br.ReadInt32();
for( int i = 0; i < size; i++ )
ch.m_letters.Add( br.ReadChar(), br.ReadInt32() );
return ch;
}
public Chain( MarkovWordGenerator gen )
{
m_letters = new Dictionary<char, int>();
m_totalSize = 0;
m_parentGen = gen;
}
