public PftRepeatableLiteral
(
PftParser.RepeatableLiteralContext context
)
{
string text = context.GetText();
Text = PftUtility.PrepareLiteral(text);
}
public PftUnconditionalLiteral
(
PftParser.UnconditionalLiteralContext context
)
{
string text = context.GetText();
Text = PftUtility.PrepareLiteral(text);
}
public PftEscapedLiteral
(
PftParser.EscapedLiteralContext context
)
{
string text = context.GetText();
Text = PftUtility.PrepareLiteral(text);
}
private static string _GetDescription
(
[NotNull] MarcRecord record
)
{
if (ReferenceEquals(_briefProgram, null))
{
string source = File.ReadAllText("sbrief_istu.pft", IrbisEncoding.Ansi);
_briefProgram = PftUtility.CompileProgram(source);
}
PftFormatter formatter = _GetFormatter(_briefProgram);
string result = formatter.FormatRecord(record);
return(result.Limit(500));
}
private static string _GetLink
(
[NotNull] MarcRecord record
)
{
if (ReferenceEquals(_linkProgram, null))
{
string source = File.ReadAllText("link.pft", IrbisEncoding.Ansi);
_linkProgram = PftUtility.CompileProgram(source);
}
PftFormatter formatter = _GetFormatter(_linkProgram);
string result = formatter.FormatRecord(record).Limit(200);
return(result.EmptyToNull());
}
private static int _GetExemplars
(
[NotNull] MarcRecord record
)
{
if (ReferenceEquals(_exemplarsProgram, null))
{
string source = File.ReadAllText("exemplars.pft", IrbisEncoding.Ansi);
_exemplarsProgram = PftUtility.CompileProgram(source);
}
PftFormatter formatter = _GetFormatter(_exemplarsProgram);
string result = formatter.FormatRecord(record);
return(result.SafeToInt32());
}
private static string _GetType
(
[NotNull] MarcRecord record
)
{
if (ReferenceEquals(_typeProgram, null))
{
string source = File.ReadAllText("type.pft", IrbisEncoding.Ansi);
_typeProgram = PftUtility.CompileProgram(source);
}
PftFormatter formatter = _GetFormatter(_typeProgram);
string formatted = formatter.FormatRecord(record);
string result = formatted.Contains("1")
? "электронный"
: "традиционный";
return(result);
}
private static string _GetAuthors
(
[NotNull] MarcRecord record
)
{
if (ReferenceEquals(_authorsProgram, null))
{
string source = File.ReadAllText("authors.pft", IrbisEncoding.Ansi);
_authorsProgram = PftUtility.CompileProgram(source);
}
PftFormatter formatter = _GetFormatter(_authorsProgram);
string merged = formatter.FormatRecord(record);
string[] lines = merged.SplitLines().NonEmptyLines().ToArray();
string result = string.Join("; ", lines).Limit(200);
return(result.EmptyToNull());
}
private static string _GetTitle
(
[NotNull] MarcRecord record
)
{
if (ReferenceEquals(_titleProgram, null))
{
string source = File.ReadAllText("title.pft");
_titleProgram = PftUtility.CompileProgram(source);
}
PftFormatter formatter = new PftFormatter
{
Program = _titleProgram
};
string result = formatter.FormatRecord(record);
return(result.Limit(250));
}
private static string _GetHeading
(
[NotNull] MarcRecord record
)
{
if (ReferenceEquals(_headingProgram, null))
{
string source = File.ReadAllText("heading.pft");
_headingProgram = PftUtility.CompileProgram(source);
}
PftFormatter formatter = new PftFormatter
{
Program = _headingProgram
};
string result = formatter.FormatRecord(record).Limit(128);
return(result.EmptyToNull());
}
static void Main(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine("Need 1 argument");
return;
}
string inputFileName = args[0];
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
try
{
stopwords = IrbisStopWords.ParseFile("IBIS.STW");
string source = File.ReadAllText("words.pft");
formatter = new PftFormatter()
{
Program = PftUtility.CompileProgram(source)
};
//DataflowLinkOptions linkOptions = new DataflowLinkOptions
//{
// PropagateCompletion = true
//};
//ExecutionDataflowBlockOptions executionOptions
// = new ExecutionDataflowBlockOptions
//{
// MaxDegreeOfParallelism = 4
//};
//processBlock = new ActionBlock<MarcRecord>
// (
// (Action<MarcRecord>)ProcessRecord,
// executionOptions
// );
using (FileStream stream = File.Create("words.bin"))
using (writer = new BinaryWriter(stream))
using (accessor = new DirectAccess64(inputFileName))
{
//maxMfn = accessor.GetMaxMfn();
maxMfn = 150000;
Console.WriteLine("Max MFN={0}", maxMfn);
// Сначала считываем все записи
for (int mfn = 1; mfn < maxMfn; mfn++)
{
ReadRecord(mfn);
}
}
using (StreamWriter textWriter = File.CreateText("words.dic"))
{
string[] keys = dictionary.Keys.ToArray();
Array.Sort(keys);
foreach (string key in keys)
{
textWriter.WriteLine("{0}\t{1}", key, dictionary[key]);
}
}
// Дожидаемся завершения
// processBlock.Complete();
// processBlock.Completion.Wait();
Console.WriteLine
(
"Good records={0}, dictionary size={1}, longest array={2}",
goodRecords,
dictionary.Count,
longest
);
DictionaryCounterInt32 <int> counter = new DictionaryCounterInt32 <int>();
using (FileStream stream = File.OpenRead("words.bin"))
using (BinaryReader reader = new BinaryReader(stream))
{
while (stream.Position < stream.Length)
{
BookData data = new BookData();
data.RestoreFromStream(reader);
foreach (int word in data.Words)
{
counter.Increment(word);
}
}
}
int maxCount = counter.Values.Max();
int threshold = maxCount / 5 + 1;
Console.WriteLine
(
"Max count={0}, threshold={1}",
maxCount,
threshold
);
using (FileStream stream = File.OpenRead("words.bin"))
using (BinaryReader reader = new BinaryReader(stream))
using (StreamWriter textWriter = File.CreateText("words.csv"))
{
while (stream.Position < stream.Length)
{
BookData data = new BookData();
data.RestoreFromStream(reader);
int i;
for (i = 0; i < data.Words.Length; i++)
{
textWriter.Write("{0},", data.Words[i]);
}
for (; i < longest; i++)
{
textWriter.Write("0,");
}
textWriter.WriteLine("{0}", data.Count);
}
}
Console.WriteLine("Complete");
}
catch (Exception exception)
{
Console.WriteLine(exception);
}
stopwatch.Stop();
TimeSpan elapsed = stopwatch.Elapsed;
Console.WriteLine("Elapsed: {0}", elapsed.ToAutoString());
}
