public async Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter)
{
Guard.NotNull(directory);
Guard.NotNull(snapshotter);
var directoryInfo = ((FSDirectory)directory).Directory;
var commit = snapshotter.Snapshot();
try
{
using (var fileStream = GetArchiveStream(directoryInfo))
{
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true))
{
foreach (var fileName in commit.FileNames)
{
var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName));
try
{
if (!file.Name.Equals(ArchiveFile, StringComparison.OrdinalIgnoreCase) &&
!file.Name.Equals(LockFile, StringComparison.OrdinalIgnoreCase))
{
zipArchive.CreateEntryFromFile(file.FullName, file.Name);
}
}
catch (IOException)
{
continue;
}
}
}
fileStream.Position = 0;
await assetStore.UploadAsync(directoryInfo.Name, fileStream, true);
}
}
finally
{
snapshotter.Release(commit);
}
}
public async Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter)
{
var directoryInfo = ((FSDirectory)directory).Directory;
var commit = snapshotter.Snapshot();
try
{
var fileId = directoryInfo.Name;
try
{
await bucket.DeleteAsync(fileId);
}
catch (GridFSFileNotFoundException)
{
}
using (var stream = await bucket.OpenUploadStreamAsync(fileId, fileId))
{
using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, true))
{
foreach (var fileName in commit.FileNames)
{
var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName));
using (var fileStream = file.OpenRead())
{
var entry = zipArchive.CreateEntry(fileStream.Name);
using (var entryStream = entry.Open())
{
await fileStream.CopyToAsync(entryStream);
}
}
}
}
}
}
finally
{
snapshotter.Release(commit);
}
}
/// <exception cref="IOException"></exception>
public virtual void Release()
{
try
{
indexSdp.Release(indexCommit);
}
finally
{
taxonomySdp.Release(taxonomyCommit);
}
try
{
indexWriter.DeleteUnusedFiles();
}
finally
{
taxonomyWriter.IndexWriter.DeleteUnusedFiles();
}
}
public async Task FlushAsync()
{
if (updates > 0 && indexWriter != null)
{
indexWriter.Commit();
indexWriter.Flush(true, true);
CleanReader();
var commit = snapshotter.Snapshot();
try
{
await assetStore.UploadDirectoryAsync(directory, commit);
}
finally
{
snapshotter.Release(commit);
}
updates = 0;
}
}
public void Backup(string backupDirectory)
{
bool hasSnapshot = false;
bool throwOnFinallyException = true;
try
{
var existingFiles = new HashSet <string>();
var allFilesPath = Path.Combine(backupDirectory, "index-files.all-existing-index-files");
var saveToFolder = Path.Combine(backupDirectory, "Indexes");
Directory.CreateDirectory(saveToFolder);
if (File.Exists(allFilesPath))
{
foreach (var file in File.ReadLines(allFilesPath))
{
existingFiles.Add(file);
}
}
var neededFilePath = Path.Combine(saveToFolder, "index-files.required-for-index-restore");
using (var allFilesWriter = File.Exists(allFilesPath) ? File.AppendText(allFilesPath) : File.CreateText(allFilesPath))
using (var neededFilesWriter = File.CreateText(neededFilePath))
{
var segmentsFileName = "segments.gen";
var segmentsFullPath = Path.Combine(indexDirectory, segmentsFileName);
File.Copy(segmentsFullPath, Path.Combine(saveToFolder, segmentsFileName));
allFilesWriter.WriteLine(segmentsFileName);
neededFilesWriter.WriteLine(segmentsFileName);
var versionFileName = "index.version";
var versionFullPath = Path.Combine(indexDirectory, versionFileName);
File.Copy(versionFullPath, Path.Combine(saveToFolder, versionFileName));
allFilesWriter.WriteLine(versionFileName);
neededFilesWriter.WriteLine(versionFileName);
var commit = snapshotter.Snapshot();
hasSnapshot = true;
foreach (var fileName in commit.FileNames)
{
var fullPath = Path.Combine(indexDirectory, fileName);
if (".lock".Equals(Path.GetExtension(fullPath), StringComparison.InvariantCultureIgnoreCase))
{
continue;
}
if (File.Exists(fullPath) == false)
{
continue;
}
if (existingFiles.Contains(fileName) == false)
{
var destFileName = Path.Combine(saveToFolder, fileName);
try
{
File.Copy(fullPath, destFileName);
}
catch (Exception e)
{
Log.WarnException(
"Could not backup RavenFS index" +
" because failed to copy file : " + fullPath + ". Skipping the index, will force index reset on restore", e); //TODO: is it also true for RavenFS?
neededFilesWriter.Dispose();
TryDelete(neededFilePath);
return;
}
allFilesWriter.WriteLine(fileName);
}
neededFilesWriter.WriteLine(fileName);
}
allFilesWriter.Flush();
neededFilesWriter.Flush();
}
}
catch
{
throwOnFinallyException = false;
throw;
}
finally
{
if (snapshotter != null && hasSnapshot)
{
try
{
snapshotter.Release();
}
catch
{
if (throwOnFinallyException)
{
throw;
}
}
}
}
}
public bool Copy(SimpleFSDirectory sourceLuceneDirectory, Analyzer analyzer, DirectoryInfo targetPath)
{
if (targetPath.Exists == false)
{
Directory.CreateDirectory(targetPath.FullName);
}
//copy index if it exists, don't do anything if it's not there
if (IndexReader.IndexExists(sourceLuceneDirectory) == false)
{
return(true);
}
using (new IndexWriter(
//read from the underlying/default directory, not the temp codegen dir
sourceLuceneDirectory,
analyzer,
_snapshotter,
IndexWriter.MaxFieldLength.UNLIMITED))
{
try
{
//var basePath = IOHelper.MapPath(configuredPath);
var commit = _snapshotter.Snapshot();
var allSnapshotFiles = commit.GetFileNames()
.Concat(new[]
{
commit.GetSegmentsFileName(),
//we need to manually include the segments.gen file
"segments.gen"
})
.Distinct()
.ToArray();
//Get all files in the temp storage that don't exist in the snapshot collection, we want to remove these
var toRemove = targetPath.GetFiles()
.Select(x => x.Name)
.Except(allSnapshotFiles);
//using (var tempDirectory = new SimpleFSDirectory(tempDir))
//{
//TODO: We're ignoring if it is locked right now, it shouldn't be unless for some strange reason the
// last process hasn't fully shut down, in that case we're not going to worry about it.
//if (IndexWriter.IsLocked(tempDirectory) == false)
//{
foreach (var file in toRemove)
{
try
{
File.Delete(Path.Combine(targetPath.FullName, file));
}
catch (IOException ex)
{
//TODO: we're ignoring this, as old files shouldn't affect the index/search operations, lucene files are 'write once'
//TODO: Return some info?
//quit here
//return false;
}
}
//}
//else
//{
// LogHelper.Warn<LocalTempStorageIndexer>("Cannot sync index files from main storage, the index is currently locked");
// //quit here
// return false;
//}
foreach (var fileName in allSnapshotFiles.Where(f => string.IsNullOrWhiteSpace(f) == false))
{
var destination = Path.Combine(targetPath.FullName, Path.GetFileName(fileName));
//don't copy if it's already there, lucene is 'write once' so this file is meant to be there already
if (File.Exists(destination))
{
continue;
}
try
{
File.Copy(
Path.Combine(sourceLuceneDirectory.GetDirectory().FullName, "Index", fileName),
destination);
}
catch (IOException ex)
{
//TODO: Return some info?
//quit here
return(false);
}
}
//}
}
finally
{
_snapshotter.Release();
}
}
return(true);
}
public virtual void Release()
{
sdp.Release(commit);
writer.DeleteUnusedFiles();
}
public void Dispose()
{
_snapshotMaker.Release();
}
