public void TestFastLoadWorksCorrectly()
{
Directory.CreateDirectory(path);
var store = new FileBasedDictionary<Dog>(path,false);
Guid id = Guid.NewGuid();
Dog dog = new Dog();
dog.Age = 20;
store[id] = dog;
// as a side effect we will have the fastload file
store.Validate();
dog.Age = 21;
store[id] = dog;
var store2 = new FileBasedDictionary<Dog>(path, false);
store2.LoadFastLoadData();
store2.Validate();
Assert.AreEqual(21, store2[id].Age);
store.Dispose();
store2.Dispose();
Directory.Delete(path, true);
}
public void SetValueAndFlushRecoversFromFailedOpenFileStream()
{
FileBasedDictionaryFileSystem fs = new FileBasedDictionaryFileSystem(
openFileStreamFailurePath: MockEntryFileName + ".tmp",
maxOpenFileStreamFailures: 5,
fileExistsFailurePath: null,
maxFileExistsFailures: 0,
maxMoveAndOverwriteFileFailures: 5);
FileBasedDictionary <string, string> dut = CreateFileBasedDictionary(fs, string.Empty);
dut.SetValueAndFlush(TestKey, TestValue);
fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(TestEntry);
}
public void SetValueAndFlushRecoversFromDeletedTempAndFailedOverwrite()
{
FileBasedDictionaryFileSystem fs = new FileBasedDictionaryFileSystem(
openFileStreamFailurePath: null,
maxOpenFileStreamFailures: 0,
fileExistsFailurePath: MockEntryFileName + ".tmp",
maxFileExistsFailures: 5,
maxMoveAndOverwriteFileFailures: 5);
FileBasedDictionary <string, string> dut = CreateFileBasedDictionary(fs, string.Empty);
dut.SetValueAndFlush(TestKey, TestValue);
this.FileBasedDictionaryFileSystemShouldContain(fs, new[] { TestEntry });
}
public void TestItemsCanBeSetCorrectly()
{
Directory.CreateDirectory(path);
var store = new FileBasedDictionary<Dog>(path);
Guid id = Guid.NewGuid();
var dog = new Dog() { Age = 100 };
store[id] = dog;
Assert.AreEqual(dog, store[id]);
store.Dispose();
Directory.Delete(path, true);
}
public void SetValuesAndFlushWritesNewEntryAndUpdatesExistingEntryOnDisk()
{
FileBasedDictionaryFileSystem fs = new FileBasedDictionaryFileSystem();
FileBasedDictionary <string, string> dut = CreateFileBasedDictionary(fs, string.Empty);
// Add TestKey to disk
dut.SetValueAndFlush(TestKey, TestValue);
fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(TestEntry);
// This call to SetValuesAndFlush should update TestKey and write TestKey2
dut.SetValuesAndFlush(
new[]
{
new KeyValuePair <string, string>(TestKey, UpdatedTestValue),
new KeyValuePair <string, string>(TestKey2, TestValue2),
});
this.FileBasedDictionaryFileSystemShouldContain(fs, new[] { UpdatedTestEntry, TestEntry2 });
}
public void TestItemsCanBeLoadedFromStore()
{
Directory.CreateDirectory(path);
var store = new FileBasedDictionary<Dog>(path);
Guid id = Guid.NewGuid();
var dog = new Dog() { Age = 100 };
store[id] = dog;
var store2 = new FileBasedDictionary<Dog>(path);
Assert.AreEqual(100, store2[id].Age);
store2.Dispose();
store.Dispose();
Directory.Delete(path, true);
}
public static bool IsNoopPrefetch(
ITracer tracer,
FileBasedDictionary <string, string> lastPrefetchArgs,
string commitId,
List <string> files,
List <string> folders,
bool hydrateFilesAfterDownload)
{
if (lastPrefetchArgs != null &&
lastPrefetchArgs.TryGetValue(PrefetchArgs.CommitId, out string lastCommitId) &&
lastPrefetchArgs.TryGetValue(PrefetchArgs.Files, out string lastFilesString) &&
lastPrefetchArgs.TryGetValue(PrefetchArgs.Folders, out string lastFoldersString) &&
lastPrefetchArgs.TryGetValue(PrefetchArgs.Hydrate, out string lastHydrateString))
{
string newFilesString = JsonConvert.SerializeObject(files);
string newFoldersString = JsonConvert.SerializeObject(folders);
bool isNoop =
commitId == lastCommitId &&
hydrateFilesAfterDownload.ToString() == lastHydrateString &&
newFilesString == lastFilesString &&
newFoldersString == lastFoldersString;
tracer.RelatedEvent(
EventLevel.Informational,
"BlobPrefetcher.IsNoopPrefetch",
new EventMetadata
{
{ "Last" + PrefetchArgs.CommitId, lastCommitId },
{ "Last" + PrefetchArgs.Files, lastFilesString },
{ "Last" + PrefetchArgs.Folders, lastFoldersString },
{ "Last" + PrefetchArgs.Hydrate, lastHydrateString },
{ "New" + PrefetchArgs.CommitId, commitId },
{ "New" + PrefetchArgs.Files, newFilesString },
{ "New" + PrefetchArgs.Folders, newFoldersString },
{ "New" + PrefetchArgs.Hydrate, hydrateFilesAfterDownload.ToString() },
{ "Result", isNoop },
});
return(isNoop);
}
return(false);
}
public void TestItemsCanBeSetCorrectly()
{
Directory.CreateDirectory(path);
var store = new FileBasedDictionary <Dog>(path);
Guid id = Guid.NewGuid();
var dog = new Dog()
{
Age = 100
};
store[id] = dog;
Assert.AreEqual(dog, store[id]);
store.Dispose();
Directory.Delete(path, true);
}
public void TestItemsForceOtherStoresToRefresh()
{
Directory.CreateDirectory(path);
var store = new FileBasedDictionary <Dog>(path);
var store2 = new FileBasedDictionary <Dog>(path);
#if (DEBUG)
//store.TrackingId = "first";
//store2.TrackingId = "second";
#endif
Guid id = Guid.NewGuid();
var dog = new Dog()
{
Age = 100, OptionalName = "milo"
};
store[id] = dog;
while (store2[id] == null)
{
Thread.Sleep(1);
}
Assert.AreEqual(100, store2[id].Age);
var dog2 = store2[id];
dog2.Age = 99;
dog2.OptionalName = "milo2";
store2[id] = dog2;
while (dog.Age != 99)
{
Thread.Sleep(1);
}
Assert.IsTrue(dog.ChangeCount > 0);
store.Dispose();
store2.Dispose();
Directory.Delete(path, true);
}
private void LoadBlobPrefetchArgs(
ITracer tracer,
GVFSEnlistment enlistment,
out string headCommitId,
out List <string> filesList,
out List <string> foldersList,
out FileBasedDictionary <string, string> lastPrefetchArgs)
{
string error;
if (!FileBasedDictionary <string, string> .TryCreate(
tracer,
Path.Combine(enlistment.DotGVFSRoot, "LastBlobPrefetch.dat"),
new PhysicalFileSystem(),
out lastPrefetchArgs,
out error))
{
tracer.RelatedWarning("Unable to load last prefetch args: " + error);
}
filesList = new List <string>();
foldersList = new List <string>();
if (!BlobPrefetcher.TryLoadFileList(enlistment, this.Files, this.FilesListFile, filesList, readListFromStdIn: this.FilesFromStdIn, error: out error))
{
this.ReportErrorAndExit(tracer, error);
}
if (!BlobPrefetcher.TryLoadFolderList(enlistment, this.Folders, this.FoldersListFile, foldersList, readListFromStdIn: this.FoldersFromStdIn, error: out error))
{
this.ReportErrorAndExit(tracer, error);
}
GitProcess gitProcess = new GitProcess(enlistment);
GitProcess.Result result = gitProcess.RevParse(GVFSConstants.DotGit.HeadName);
if (result.ExitCodeIsFailure)
{
this.ReportErrorAndExit(tracer, result.Errors);
}
headCommitId = result.Output.Trim();
}
public string [] GetCompletionList(string prefixText, int count)
{
var fileBasedDictionary = new FileBasedDictionary(Server);
try
{
using (Stream ForwardDict = fileBasedDictionary.OpenForwardIndex())
{
var dict = DawgSharp.Dawg <bool> .Load(ForwardDict, r => r.ReadBoolean());
return(dict.MatchPrefix(DictionaryHelper.RemoveStressMarks(prefixText).ToLowerInvariant()).Take(10).Select(kvp => kvp.Key).ToArray());
}
}
catch (Exception exception)
{
Email.SendAdminEmail("GetCompletionList", exception.ToString());
return(new [] { "Доступ к словарю в данный момент отсутствует. Возможно происходит построение индексов." });
}
}
private static FileBasedDictionary <string, string> CreateFileBasedDictionary(FileBasedDictionaryFileSystem fs, string initialContents)
{
fs.ExpectedFiles.Add(MockEntryFileName, new ReusableMemoryStream(initialContents));
fs.ExpectedOpenFileStreams.Add(MockEntryFileName + ".tmp", new ReusableMemoryStream(string.Empty));
fs.ExpectedOpenFileStreams.Add(MockEntryFileName, fs.ExpectedFiles[MockEntryFileName]);
string error;
FileBasedDictionary <string, string> dut;
FileBasedDictionary <string, string> .TryCreate(null, MockEntryFileName, fs, out dut, out error).ShouldEqual(true, error);
dut.ShouldNotBeNull();
// FileBasedDictionary should only open a file stream to the non-tmp file when being created. At all other times it should
// write to a tmp file and overwrite the non-tmp file
fs.ExpectedOpenFileStreams.Remove(MockEntryFileName);
return(dut);
}
public void SetValuesAndFlushWritesUpdatesExistingEntriesOnDisk()
{
FileBasedDictionaryFileSystem fs = new FileBasedDictionaryFileSystem();
FileBasedDictionary <string, string> dut = CreateFileBasedDictionary(fs, string.Empty);
dut.SetValuesAndFlush(
new[]
{
new KeyValuePair <string, string>(TestKey, TestValue),
new KeyValuePair <string, string>(TestKey2, TestValue2),
});
this.FileBasedDictionaryFileSystemShouldContain(fs, new[] { TestEntry, TestEntry2 });
dut.SetValuesAndFlush(
new[]
{
new KeyValuePair <string, string>(TestKey, UpdatedTestValue),
new KeyValuePair <string, string>(TestKey2, UpdatedTestValue2),
});
this.FileBasedDictionaryFileSystemShouldContain(fs, new[] { UpdatedTestEntry, UpdatedTestEntry2 });
}
public void SetValuesAndFlushWritesUpdatesExistingEntriesOnDisk()
{
FileBasedDictionaryFileSystem fs = new FileBasedDictionaryFileSystem();
FileBasedDictionary <string, string> dut = CreateFileBasedDictionary(fs, string.Empty);
dut.SetValuesAndFlush(
new[]
{
new KeyValuePair <string, string>(TestKey, TestValue),
new KeyValuePair <string, string>(TestKey2, TestValue2),
});
fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(TestEntry + TestEntry2);
dut.SetValuesAndFlush(
new[]
{
new KeyValuePair <string, string>(TestKey, UpdatedTestValue),
new KeyValuePair <string, string>(TestKey2, UpdatedTestValue2),
});
fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(UpdatedTestEntry + UpdatedTestEntry2);
}
public void TestItemsCanBeLoadedFromStore()
{
Directory.CreateDirectory(path);
var store = new FileBasedDictionary <Dog>(path);
Guid id = Guid.NewGuid();
var dog = new Dog()
{
Age = 100
};
store[id] = dog;
var store2 = new FileBasedDictionary <Dog>(path);
Assert.AreEqual(100, store2[id].Age);
store2.Dispose();
store.Dispose();
Directory.Delete(path, true);
}
private SqliteItemRepository(string dbPath)
{
Logger.ReportInfo("==========Using new SQL Repo========");
dbFileName = dbPath;
if (!ConnectToDB(dbPath)) throw new ApplicationException("CRITICAL ERROR - Unable to connect to database: " + dbPath + ". Program cannot function.");
displayRepo = new SQLiteDisplayRepository(Path.Combine(ApplicationPaths.AppUserSettingsPath, "display.db"));
using (new MediaBrowser.Util.Profiler("Playstate initialization"))
playbackStatus = new FileBasedDictionary<PlaybackStatus>(GetPath("playstate", ApplicationPaths.AppUserSettingsPath));
//string playStateDBPath = Path.Combine(ApplicationPaths.AppUserSettingsPath, "playstate.db");
string[] queries = {"create table if not exists provider_data (guid, full_name, data)",
"create unique index if not exists idx_provider on provider_data(guid, full_name)",
"create table if not exists items (guid primary key)",
"create index if not exists idx_items on items(guid)",
"create table if not exists children (guid, child)",
"create unique index if not exists idx_children on children(guid, child)",
"create table if not exists list_items(guid, property, value, sort_order)",
"create index if not exists idx_list on list_items(guid, property)",
"create unique index if not exists idx_list_constraint on list_items(guid, property, value)",
"create table if not exists schema_version (table_name primary key, version)",
//triggers
triggerSQL,
//pragmas
"pragma temp_store = memory",
// @"create table display_prefs (guid primary key, view_type, show_labels, vertical_scroll
// sort_order, index_by, use_banner, thumb_constraint_width, thumb_constraint_height, use_coverflow, use_backdrop )"
//, "create table play_states (guid primary key, play_count, position_ticks, playlist_position, last_played)"
};
RunQueries(queries);
alive = true; // tell writer to keep going
Async.Queue("Sqlite Writer", DelayedWriter);
}
private void PrefetchBlobs(
ITracer tracer,
GVFSEnlistment enlistment,
string headCommitId,
List <string> filesList,
List <string> foldersList,
FileBasedDictionary <string, string> lastPrefetchArgs,
GitObjectsHttpRequestor objectRequestor,
CacheServerInfo cacheServer)
{
BlobPrefetcher blobPrefetcher = new BlobPrefetcher(
tracer,
enlistment,
objectRequestor,
filesList,
foldersList,
lastPrefetchArgs,
ChunkSize,
SearchThreadCount,
DownloadThreadCount,
IndexThreadCount);
if (blobPrefetcher.FolderList.Count == 0 &&
blobPrefetcher.FileList.Count == 0)
{
this.ReportErrorAndExit(tracer, "Did you mean to fetch all blobs? If so, specify `--files '*'` to confirm.");
}
if (this.HydrateFiles)
{
if (!this.CheckIsMounted(verbose: true))
{
this.ReportErrorAndExit("You can only specify --hydrate if the repo is mounted. Run 'gvfs mount' and try again.");
}
}
int matchedBlobCount = 0;
int downloadedBlobCount = 0;
int hydratedFileCount = 0;
Func <bool> doPrefetch =
() =>
{
try
{
blobPrefetcher.PrefetchWithStats(
headCommitId,
isBranch: false,
hydrateFilesAfterDownload: this.HydrateFiles,
matchedBlobCount: out matchedBlobCount,
downloadedBlobCount: out downloadedBlobCount,
hydratedFileCount: out hydratedFileCount);
return(!blobPrefetcher.HasFailures);
}
catch (BlobPrefetcher.FetchException e)
{
tracer.RelatedError(e.Message);
return(false);
}
};
if (this.Verbose)
{
doPrefetch();
}
else
{
string message =
this.HydrateFiles
? "Fetching blobs and hydrating files "
: "Fetching blobs ";
this.ShowStatusWhileRunning(doPrefetch, message + this.GetCacheServerDisplay(cacheServer, enlistment.RepoUrl));
}
if (blobPrefetcher.HasFailures)
{
Environment.ExitCode = 1;
}
else
{
Console.WriteLine();
Console.WriteLine("Stats:");
Console.WriteLine(" Matched blobs: " + matchedBlobCount);
Console.WriteLine(" Already cached: " + (matchedBlobCount - downloadedBlobCount));
Console.WriteLine(" Downloaded: " + downloadedBlobCount);
if (this.HydrateFiles)
{
Console.WriteLine(" Hydrated files: " + hydratedFileCount);
}
}
}
public void TestMethod1()
{
Assert.AreEqual("Анна,жо,Анны,Анне,Анну,Анной,Анне,Анне,Анны,Анн,Аннам,Анн,Аннами,Аннах,Аннах,Анною",
FileBasedDictionary.ExpandLine("Анна 1 жо 1а"));
}
public void TestItemsForceOtherStoresToRefresh()
{
Directory.CreateDirectory(path);
var store = new FileBasedDictionary<Dog>(path);
var store2 = new FileBasedDictionary<Dog>(path);
#if (DEBUG)
store.TrackingId = "first";
store2.TrackingId = "second";
#endif
Guid id = Guid.NewGuid();
var dog = new Dog() { Age = 100, OptionalName = "milo" };
store[id] = dog;
while (store2[id] == null) {
Thread.Sleep(1);
}
Assert.AreEqual(100, store2[id].Age);
var dog2 = store2[id];
dog2.Age = 99;
dog2.OptionalName = "milo2";
store2[id] = dog2;
while (dog.Age != 99) {
Thread.Sleep(1);
}
Assert.IsTrue(dog.ChangeCount > 0);
store.Dispose();
store2.Dispose();
Directory.Delete(path, true);
}
public ItemRepository()
{
playbackStatus = new FileBasedDictionary <PlaybackStatus>(GetPath("playstate", userSettingPath));
}
public void GenerateCsv()
{
FileBasedDictionary.GenerateCsv(@"..\..\..\..\zalizniak\zalizniak.txt", "odict.csv");
}
public ItemRepository() {
playbackStatus = new FileBasedDictionary<PlaybackStatus>(GetPath("playstate", userSettingPath));
thumbSizes = new FileBasedDictionary<ThumbSize>(GetPath("thumbsizes", userSettingPath));
}
private SqliteItemRepository(string dbPath)
{
Logger.ReportInfo("==========Using new SQL Repo========");
dbFileName = dbPath;
SQLiteConnectionStringBuilder connectionstr = new SQLiteConnectionStringBuilder();
connectionstr.PageSize = 4096;
connectionstr.CacheSize = 4096;
connectionstr.SyncMode = SynchronizationModes.Normal;
connectionstr.DataSource = dbPath;
connectionstr.JournalMode = SQLiteJournalModeEnum.Delete;
connection = new SQLiteConnection(connectionstr.ConnectionString);
int retries = 0;
bool connected = false;
while (!connected && retries < MAX_RETRIES)
{
try
{
connection.Open();
connected = true;
}
catch (Exception e)
{
Logger.ReportException("Error connecting to database! Will retry "+MAX_RETRIES+" times.", e);
retries++;
Thread.Sleep(250);
}
}
if (!connected) throw new ApplicationException("CRITICAL ERROR - Unable to connect to database: " + dbPath + ". Program cannot function.");
displayRepo = new SQLiteDisplayRepository(Path.Combine(ApplicationPaths.AppUserSettingsPath, "display.db"));
playbackStatus = new FileBasedDictionary<PlaybackStatus>(GetPath("playstate", ApplicationPaths.AppUserSettingsPath));
//string playStateDBPath = Path.Combine(ApplicationPaths.AppUserSettingsPath, "playstate.db");
string[] queries = {"create table if not exists provider_data (guid, full_name, data)",
"create unique index if not exists idx_provider on provider_data(guid, full_name)",
"create table if not exists items (guid primary key)",
"create index if not exists idx_items on items(guid)",
"create table if not exists children (guid, child)",
"create unique index if not exists idx_children on children(guid, child)",
"create table if not exists list_items(guid, property, value, sort_order)",
"create index if not exists idx_list on list_items(guid, property)",
"create unique index if not exists idx_list_constraint on list_items(guid, property, value)",
"create table if not exists schema_version (table_name primary key, version)",
//"create table if not exists recent_list(top_parent, child, date_added)",
//"create index if not exists idx_recent on recent_list(top_parent, child)",
//"attach database '"+playStateDBPath+"' as playstate_db",
//"create table if not exists playstate_db.play_states (guid primary key, play_count, position_ticks, playlist_position, last_played)",
"pragma temp_store = memory",
// @"create table display_prefs (guid primary key, view_type, show_labels, vertical_scroll
// sort_order, index_by, use_banner, thumb_constraint_width, thumb_constraint_height, use_coverflow, use_backdrop )"
//, "create table play_states (guid primary key, play_count, position_ticks, playlist_position, last_played)"
};
foreach (var query in queries) {
try {
connection.Exec(query);
} catch (Exception e) {
Logger.ReportInfo(e.ToString());
}
}
alive = true; // tell writer to keep going
Async.Queue("Sqlite Writer", DelayedWriter);
}
