private static void ProcessOneDatabase
(
[NotNull] string inputFileName
)
{
using (DirectAccess64 accessor
= new DirectAccess64(inputFileName, DirectAccessMode.ReadOnly))
{
int maxMfn = accessor.GetMaxMfn();
Console.WriteLine("{0}: Max MFN={1}", inputFileName, maxMfn);
for (int mfn = 1; mfn < maxMfn; mfn++)
{
if (mfn % 100 == 1)
{
Console.Write(" {0} ", mfn - 1);
}
try
{
MarcRecord record = accessor.ReadRecord(mfn);
if (!ReferenceEquals(record, null))
{
ReaderInfo readerInfo = ReaderInfo.Parse(record);
string ticket = readerInfo.Ticket;
if (string.IsNullOrEmpty(ticket))
{
continue;
}
DepersonalizedReader depersonalized;
if (readers.TryGetValue(ticket, out depersonalized))
{
depersonalized.AddVisits(readerInfo.Visits);
}
else
{
depersonalized = DepersonalizedReader
.FromReaderInfo(readerInfo);
readers.Add(ticket, depersonalized);
}
}
}
catch (Exception exception)
{
Console.WriteLine();
Console.WriteLine("MFN {0}: {1}", mfn, exception.Message);
}
}
}
Console.WriteLine("Total: {0}", readers.Count);
Console.WriteLine("Memory: {0}", Process.GetCurrentProcess().WorkingSet64);
}
static void Main(string[] args)
{
if (args.Length != 1)
{
return;
}
try
{
DirectAccess64 irbis = new DirectAccess64(args[0]);
int maxMfn = irbis.GetMaxMfn();
Console.WriteLine("Max MFN={0}", maxMfn);
Parallel.For(1, maxMfn, mfn =>
{
if (mfn % 1000 == 1)
{
Console.Write(".");
}
try
{
MarcRecord record = irbis.ReadRecord(mfn);
if (!ReferenceEquals(record, null))
{
ProcessRecord(record);
}
}
catch
{
// Do nothing
}
});
irbis.Dispose();
using (StreamWriter writer = File.CreateText("publishers.txt"))
{
string[] keys = counter1.Keys.OrderBy(_ => _).ToArray();
foreach (string key in keys)
{
writer.WriteLine("{0}\t{1}\t{2}", key, counter1[key],
counter2[key]);
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
static void Main(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine("Need 1 argument");
return;
}
string inputFileName = args[0];
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
try
{
stopwords = IrbisStopWords.ParseFile("IBIS.STW");
string source = File.ReadAllText("words.pft");
formatter = new PftFormatter()
{
Program = PftUtility.CompileProgram(source)
};
//DataflowLinkOptions linkOptions = new DataflowLinkOptions
//{
// PropagateCompletion = true
//};
//ExecutionDataflowBlockOptions executionOptions
// = new ExecutionDataflowBlockOptions
//{
// MaxDegreeOfParallelism = 4
//};
//processBlock = new ActionBlock<MarcRecord>
// (
// (Action<MarcRecord>)ProcessRecord,
// executionOptions
// );
using (FileStream stream = File.Create("words.bin"))
using (writer = new BinaryWriter(stream))
using (accessor = new DirectAccess64(inputFileName))
{
//maxMfn = accessor.GetMaxMfn();
maxMfn = 150000;
Console.WriteLine("Max MFN={0}", maxMfn);
// Сначала считываем все записи
for (int mfn = 1; mfn < maxMfn; mfn++)
{
ReadRecord(mfn);
}
}
using (StreamWriter textWriter = File.CreateText("words.dic"))
{
string[] keys = dictionary.Keys.ToArray();
Array.Sort(keys);
foreach (string key in keys)
{
textWriter.WriteLine("{0}\t{1}", key, dictionary[key]);
}
}
// Дожидаемся завершения
// processBlock.Complete();
// processBlock.Completion.Wait();
Console.WriteLine
(
"Good records={0}, dictionary size={1}, longest array={2}",
goodRecords,
dictionary.Count,
longest
);
DictionaryCounterInt32 <int> counter = new DictionaryCounterInt32 <int>();
using (FileStream stream = File.OpenRead("words.bin"))
using (BinaryReader reader = new BinaryReader(stream))
{
while (stream.Position < stream.Length)
{
BookData data = new BookData();
data.RestoreFromStream(reader);
foreach (int word in data.Words)
{
counter.Increment(word);
}
}
}
int maxCount = counter.Values.Max();
int threshold = maxCount / 5 + 1;
Console.WriteLine
(
"Max count={0}, threshold={1}",
maxCount,
threshold
);
using (FileStream stream = File.OpenRead("words.bin"))
using (BinaryReader reader = new BinaryReader(stream))
using (StreamWriter textWriter = File.CreateText("words.csv"))
{
while (stream.Position < stream.Length)
{
BookData data = new BookData();
data.RestoreFromStream(reader);
int i;
for (i = 0; i < data.Words.Length; i++)
{
textWriter.Write("{0},", data.Words[i]);
}
for (; i < longest; i++)
{
textWriter.Write("0,");
}
textWriter.WriteLine("{0}", data.Count);
}
}
Console.WriteLine("Complete");
}
catch (Exception exception)
{
Console.WriteLine(exception);
}
stopwatch.Stop();
TimeSpan elapsed = stopwatch.Elapsed;
Console.WriteLine("Elapsed: {0}", elapsed.ToAutoString());
}
static void Main(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine("Usage: DirectWriterBenchmark <path-to-MST>");
return;
}
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
string masterPath = args[0];
try
{
DirectUtility.CreateDatabase64(masterPath);
using (DirectAccess64 accessor = new DirectAccess64(masterPath))
{
for (int i = 0; i < 10000; i++)
{
if (i % 1000 == 0)
{
Console.Write(".");
}
MarcRecord record = new MarcRecord();
record.BeginUpdate(100);
for (int tag = 200; tag < 300; tag++)
{
record.Fields.Add
(
new RecordField
(
tag,
"Это поле номер " + tag
)
);
}
record.EndUpdate();
accessor.WriteRecord(record);
}
Console.WriteLine();
for (int approach = 0; approach < 10; approach++)
{
Console.WriteLine("Approach {0}", approach + 1);
for (int mfn = 1; mfn <= 1000; mfn++)
{
MarcRecord record = accessor.ReadRecord(mfn)
.ThrowIfNull("accessor.ReadRecord(mfn)");
record.Fields.Add
(
new RecordField
(
300,
"Запись отредактирована " + DateTime.Now
)
);
accessor.WriteRecord(record);
}
}
}
stopwatch.Stop();
Console.WriteLine
(
"Elapsed: {0}",
stopwatch.Elapsed
);
}
catch (Exception exception)
{
Console.WriteLine(exception);
}
}
static void Main(string[] args)
{
if (args.Length != 1)
{
Console.WriteLine("Usage: DirectReaderBenchmark <path-to-MST>");
return;
}
string masterPath = args[0];
try
{
IrbisEncoding.RelaxUtf8();
Console.WriteLine("Open: {0}", masterPath);
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
using (DirectAccess64 reader = new DirectAccess64(masterPath))
{
int maxMfn = reader.GetMaxMfn();
Console.WriteLine("Max MFN={0}", maxMfn);
Parallel.For
(
1,
maxMfn,
mfn =>
{
try
{
MarcRecord record = reader.ReadRecord(mfn);
//Console.Write('.');
if (!ReferenceEquals(record, null))
{
Debug.Assert(record.Mfn == mfn);
}
}
catch (Exception exception)
{
Console.WriteLine
(
"MFN={0}: exception: {1}",
mfn,
exception
);
}
}
);
}
stopwatch.Stop();
Console.WriteLine();
Console.WriteLine();
Console.WriteLine("Close: {0}", masterPath);
Console.WriteLine
(
"Elapsed: {0} sec",
stopwatch.Elapsed.ToSecondString()
);
}
catch (Exception exception)
{
Console.WriteLine(exception);
}
}
