public void Run(string[] args)
{
var linksFile = ConsoleHelpers.GetOrReadArgument(0, "Links file", args);
var fileToIndex = ConsoleHelpers.GetOrReadArgument(1, "File to index", args);
if (!File.Exists(linksFile))
Console.WriteLine("Entered links file does not exists.");
else if (!File.Exists(fileToIndex))
Console.WriteLine("Entered file to index does not exists.");
else
{
var cancellationSource = ConsoleHelpers.HandleCancellation();
using (var memoryManager = new UInt64LinksMemoryManager(linksFile, UInt64LinksMemoryManager.DefaultLinksSizeStep * 16))
using (var links = new UInt64Links(memoryManager))
{
var syncLinks = new SynchronizedLinks<ulong>(links);
UnicodeMap.InitNew(syncLinks);
var sequences = new Sequences(syncLinks);
var fileIndexer = new FileIndexer(syncLinks, sequences);
//fileIndexer.IndexAsync(fileToIndex, cancellationSource.Token).Wait();
fileIndexer.IndexSync(fileToIndex, cancellationSource.Token);
}
}
ConsoleHelpers.PressAnyKeyToContinue();
}
private ulong Calculate(int start, int steps)
{
const int mb4 = 4 * 1024 * 1024;
using (var memory = new HeapResizableDirectMemory(mb4))
using (var memoryManager = new UInt64LinksMemoryManager(memory))
using (var links = new UInt64Links(memoryManager))
{
ulong[] digits = new ulong[10];
SetPoints(links, digits);
SetSteps(links, digits);
ulong current = digits[start];
var counter = new Counter();
CalculateCore(links, current, steps, counter);
return counter.Count;
}
}
public static void Test(string filename)
{
//try
{
using (var memoryManager = new UInt64LinksMemoryManager(filename, 512 * 1024 * 1024))
using (var links = new UInt64Links(memoryManager))
{
var syncLinks = new SynchronizedLinks<ulong>(links);
//links.EnterTransaction();
var link = memoryManager.AllocateLink();
memoryManager.FreeLink(link);
Console.ReadKey();
var temp1 = syncLinks.Create();
var temp2 = syncLinks.Create();
var temp3 = syncLinks.CreateAndUpdate(temp1, temp2);
var temp4 = syncLinks.CreateAndUpdate(temp1, temp3);
var temp5 = syncLinks.CreateAndUpdate(temp4, temp2);
//links.Delete(links.GetSource(temp2), links.GetTarget(temp2));
//links.Each(0, temp2, x => links.PrintLink(x));
syncLinks.Each(syncLinks.Constants.Any, syncLinks.Constants.Any, x =>
{
memoryManager.PrintLink(x);
return true;
});
//links.ExportSourcesTree(filename + ".gexf");
Console.WriteLine("---");
Console.WriteLine(syncLinks.Count());
var sequences = new Sequences(syncLinks);
//var seq = sequences.Create(temp1, temp5, temp2, temp1, temp2); //, temp5);
var sequence = sequences.Create(temp1, temp5, temp2, temp1, temp2, temp3, temp2, temp4, temp1,
temp5);
//, temp5);
//links.Each(0, 0, (x, isAPoint) => { links.PrintLink(x); return true; });
//sequences.Each((x, isAPoint) => { links.PrintLink(x); return true; }, temp1, temp5, temp2, temp1, temp2, temp3, temp2, temp4, temp1, temp5);
var sequencesCount = 0;
sequences.Each(x =>
{
sequencesCount++;
return true;
}, temp1, temp5, temp2, temp1, temp2, temp3, temp2, temp4, temp1, temp5);
sequences.Compact(temp1, temp5, temp2, temp1, temp2, temp3, temp2, temp4, temp1, temp5);
Console.WriteLine(sequencesCount);
Console.WriteLine(syncLinks.Count());
sequences.Create(temp1, temp1, temp1, temp1, temp1, temp1, temp1, temp1, temp1, temp1, temp1, temp1,
temp1);
Console.WriteLine(syncLinks.Count());
Console.ReadKey();
//var ps = (from Pair score in links
// where score.Target == temp2
// select score).ToArray();
//var ls = (from Link score in links
// where score.Target == temp2
// select score).ToArray();
//links.Execute(db => from User user in links
// select user);
//var firstLink = links.First();
//links.Delete(ref firstLink);
Console.WriteLine("---");
syncLinks.Each(syncLinks.Constants.Any, syncLinks.Constants.Any, x =>
{
memoryManager.PrintLink(x);
return true;
});
Console.WriteLine("---");
//links.ExitTransaction();
//links.EnterTransaction();
//links.ExitTransaction();
}
;
}
//catch (Exception ex)
{
// ex.WriteToConsole();
}
Console.ReadKey();
}
public void Bug1Test()
{
var tempDatabaseFilename = Path.GetTempFileName();
var tempTransactionLogFilename = Path.GetTempFileName();
var itself = Constants.Itself;
// User Code Error (Autoreverted), some data saved
try
{
ulong l1;
ulong l2;
using (var memoryManager = new UInt64LinksMemoryManager(tempDatabaseFilename))
using (var links = new UInt64Links(memoryManager, tempTransactionLogFilename))
{
l1 = links.CreateAndUpdate(itself, itself);
l2 = links.CreateAndUpdate(itself, itself);
l2 = links.Update(l2, l2, l1, l2);
links.CreateAndUpdate(l2, itself);
links.CreateAndUpdate(l2, itself);
}
Global.Trash = FileHelpers.ReadAll<UInt64Links.Transition>(tempTransactionLogFilename);
using (var memoryManager = new UInt64LinksMemoryManager(tempDatabaseFilename))
using (var links = new UInt64Links(memoryManager, tempTransactionLogFilename))
{
using (var transaction = links.BeginTransaction())
{
l2 = links.Update(l2, l1);
links.Delete(l2);
ExceptionThrower();
transaction.Commit();
}
Global.Trash = links.Count();
}
}
catch
{
Global.Trash = FileHelpers
.ReadAll<UInt64Links.Transition>(tempTransactionLogFilename);
}
File.Delete(tempDatabaseFilename);
File.Delete(tempTransactionLogFilename);
}
public void TransactionDamage()
{
var itself = Constants.Itself;
var tempDatabaseFilename = Path.GetTempFileName();
var tempTransactionLogFilename = Path.GetTempFileName();
// Commit
using (var memoryManager = new UInt64LinksMemoryManager(tempDatabaseFilename))
using (var links = new UInt64Links(memoryManager, tempTransactionLogFilename))
{
using (var transaction = links.BeginTransaction())
{
var l1 = links.CreateAndUpdate(itself, itself);
var l2 = links.CreateAndUpdate(itself, itself);
Global.Trash = links.Update(l2, l2, l1, l2);
links.Delete(l1);
transaction.Commit();
}
Global.Trash = links.Count();
}
Global.Trash = FileHelpers.ReadAll<UInt64Links.Transition>(tempTransactionLogFilename);
// Damage database
FileHelpers
.WriteFirst(tempTransactionLogFilename, new UInt64Links.Transition { TransactionId = 555 });
// Try load damaged database
try
{
// TODO: Fix
using (var memoryManager = new UInt64LinksMemoryManager(tempDatabaseFilename))
using (var links = new UInt64Links(memoryManager, tempTransactionLogFilename))
{
Global.Trash = links.Count();
}
}
catch (NotSupportedException ex)
{
Assert.True(ex.Message == "Database is damaged, autorecovery is not supported yet.");
}
Global.Trash = FileHelpers
.ReadAll<UInt64Links.Transition>(tempTransactionLogFilename);
File.Delete(tempDatabaseFilename);
File.Delete(tempTransactionLogFilename);
}
public void TransactionCommit()
{
var itself = Constants.Itself;
var tempDatabaseFilename = Path.GetTempFileName();
var tempTransactionLogFilename = Path.GetTempFileName();
// Commit
using (var memoryManager = new UInt64LinksMemoryManager(tempDatabaseFilename))
using (var links = new UInt64Links(memoryManager, tempTransactionLogFilename))
{
using (var transaction = links.BeginTransaction())
{
var l1 = links.CreateAndUpdate(itself, itself);
var l2 = links.CreateAndUpdate(itself, itself);
Global.Trash = links.Update(l2, l2, l1, l2);
links.Delete(l1);
transaction.Commit();
}
Global.Trash = links.Count();
}
Global.Trash = FileHelpers.ReadAll<UInt64Links.Transition>(tempTransactionLogFilename);
}
public Transaction(UInt64Links links)
{
_lock = TransactionLocks.GetOrAdd(links, x => new ReaderWriterLockSlim(LockRecursionPolicy.NoRecursion));
_lock.EnterWriteLock(); // Самый простой случай: разрешать только 1 транзакцию базу данных
_links = links;
_commited = false;
_reverted = false;
_transitions = new ConcurrentQueue<Transition>();
// Set Current Transaction
_links._currentTransactionId = _links._lastCommitedTransition.TransactionId + 1;
_links._currentTransactionTransitions = _transitions;
}
public static void Test()
{
File.Delete("web.links");
using (var memoryManager = new UInt64LinksMemoryManager("web.links", 8 * 1024 * 1024))
using (var links = new UInt64Links(memoryManager))
{
var syncLinks = new SynchronizedLinks<ulong>(links);
UnicodeMap.InitNew(syncLinks);
var sequences = new Sequences(syncLinks);
// Get content
const string url = "https://en.wikipedia.org/wiki/Main_Page";
var pageContents = GetPageContents(url);
var totalChars = url.Length + pageContents.Length;
Global.Trash = totalChars;
var urlLink = sequences.CreateBalancedVariant(UnicodeMap.FromStringToLinkArray(url));
var responseSourceArray = UnicodeMap.FromStringToLinkArray(pageContents);
//for (var i = 0; i < 1; i++)
//{
// var sw01 = Stopwatch.StartNew();
// var responseLink = sequences.CreateBalancedVariant(responseSourceArray);
// sw01.Stop();
// Console.WriteLine(sw01.Elapsed);
//}
//var sw0 = Stopwatch.StartNew();
//var groups = UnicodeMap.FromStringToLinkArrayGroups(response);
//var responseLink = sequences.CreateBalancedVariant(groups); sw0.Stop();
//var sw1 = Stopwatch.StartNew();
//var responseCompressedArray1 = links.PrecompressSequence1(responseSourceArray); sw1.Stop();
//var sw2 = Stopwatch.StartNew();
//var responseCompressedArray2 = links.PrecompressSequence2(responseSourceArray); sw2.Stop();
// [+] Можно попробовать искать не максимальный, а первый, который встречается как минимум дважды - медленно, высокое качество, не наивысшее
// [+] Или использовать не локальный словарь, а глобальный (т.е. считать один раз, потом только делать замены) - быстро, но качество низкое
// Precompress0 - лучшее соотношение скорость / качество. (тоже что и Data.Core.Sequences.Compressor.Precompress)
ulong[] responseCompressedArray3 = null;
for (var i = 0; i < 1; i++)
{
var sw3 = Stopwatch.StartNew();
var compressor = new Data.Core.Sequences.Compressor(syncLinks, sequences, 1);
responseCompressedArray3 = compressor.Precompress(responseSourceArray); sw3.Stop();
Console.WriteLine(sw3.Elapsed);
}
// Combine Groups and Compression (first Compression, then Groups) (DONE)
// Как после сжатия не группируй, больше сжатия не получить (странно, но это факт)
//var groups = UnicodeMap.FromLinkArrayToLinkArrayGroups(responseCompressedArray3);
//var responseLink2 = sequences.CreateBalancedVariant(groups);
// Equal to `var responseLink2 = sequences.CreateBalancedVariant(responseCompressedArray3);`
//for (int i = 0; i < responseCompressedArray1.Length; i++)
//{
// if (responseCompressedArray1[i] != responseCompressedArray2[i])
// {
// }
//}
//var responseLink1 = sequences.CreateBalancedVariant(responseCompressedArray1);
var responseLink2 = sequences.CreateBalancedVariant(responseCompressedArray3);
//var decompress1 = sequences.FormatSequence(responseLink1);
var decompress2 = sequences.FormatSequence(responseLink2);
Global.Trash = decompress2;
//for (int i = 0; i < decompress1.Length; i++)
//{
// if (decompress1[i] != decompress2[i])
// {
// }
//}
var unpack = UnicodeMap.FromSequenceLinkToString(responseLink2, syncLinks);
Global.Trash = (unpack == pageContents);
var totalLinks = syncLinks.Count() - UnicodeMap.MapSize;
Console.WriteLine(totalLinks);
Global.Trash = totalLinks;
syncLinks.CreateAndUpdate(urlLink, responseLink2);
var divLinksArray = UnicodeMap.FromStringToLinkArray("div");
var fullyMatched = sequences.GetAllMatchingSequences1(divLinksArray);
var partiallyMatched = sequences.GetAllPartiallyMatchingSequences1(divLinksArray);
var intersection = fullyMatched.Intersect(partiallyMatched);
}
Console.ReadKey();
}
public static void Stats()
{
// Get content
const string url = "https://en.wikipedia.org/wiki/Main_Page";
var pageContents = GetPageContents(url);
var responseSourceArray = UnicodeMap.FromStringToLinkArray(pageContents);
for (var i = 0; i < 3; i++)
{
File.Delete("stats.links");
using (var memoryManager = new UInt64LinksMemoryManager("stats.links", 8 * 1024 * 1024))
using (var links = new UInt64Links(memoryManager))
{
var syncLinks = new SynchronizedLinks<ulong>(links);
UnicodeMap.InitNew(syncLinks);
var sequences = new Sequences(syncLinks);
var sw3 = Stopwatch.StartNew(); sequences.CreateBalancedVariant(responseSourceArray); sw3.Stop();
var totalLinks = syncLinks.Count() - UnicodeMap.MapSize;
Console.WriteLine($"Balanced Variant: {sw3.Elapsed}, {responseSourceArray.Length}, {totalLinks}");
}
}
var minFrequency = 0UL;
for (var i = 1; i < 200; i++)
{
minFrequency += (ulong)(1 + Math.Log(i));
File.Delete("stats.links");
using (var memoryManager = new UInt64LinksMemoryManager("stats.links", 8 * 1024 * 1024))
using (var links = new UInt64Links(memoryManager))
{
var syncLinks = new SynchronizedLinks<ulong>(links);
UnicodeMap.InitNew(syncLinks);
var sequences = new Sequences(syncLinks);
var sw3 = Stopwatch.StartNew();
var compressor = new Data.Core.Sequences.Compressor(syncLinks, sequences, minFrequency);
var responseCompressedArray3 = compressor.Precompress(responseSourceArray);
sequences.CreateBalancedVariant(responseCompressedArray3); sw3.Stop();
var totalLinks = syncLinks.Count() - UnicodeMap.MapSize;
Console.WriteLine($"{sw3.Elapsed}, {minFrequency}, {responseSourceArray.Length}, {totalLinks}");
}
}
Console.ReadKey();
}