public async Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter) { var directoryInfo = ((FSDirectory)directory).Directory; var commit = snapshotter.Snapshot(); try { using (var fileStream = GetArchiveStream(directoryInfo)) { using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true)) { foreach (var fileName in commit.FileNames) { var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName)); zipArchive.CreateEntryFromFile(file.FullName, file.Name); } } fileStream.Position = 0; await assetStore.UploadAsync(directoryInfo.Name, fileStream, true); } } finally { snapshotter.Release(commit); } }
public IndexSnapshot(SnapshotDeletionPolicy snapshotMaker) { _snapshotMaker = snapshotMaker; var indexCommitPoint = _snapshotMaker.Snapshot(); FileNames = indexCommitPoint.GetFileNames().ToArray(); SegmentFileName = indexCommitPoint.GetSegmentsFileName(); }
/// <summary> /// Constructor over the given <see cref="IndexWriter"/>. Uses the last /// <see cref="IndexCommit"/> found in the <see cref="Directory"/> managed by the given /// writer. /// </summary> public IndexRevision(IndexWriter writer) { sdp = writer.Config.IndexDeletionPolicy as SnapshotDeletionPolicy; if (sdp == null) { throw new ArgumentException("IndexWriter must be created with SnapshotDeletionPolicy", "writer"); } this.writer = writer; this.commit = sdp.Snapshot(); this.version = RevisionVersion(commit); this.sourceFiles = RevisionFiles(commit); }
/// <summary> /// Constructor over the given <see cref="IndexWriter"/>. Uses the last /// <see cref="IndexCommit"/> found in the <see cref="Directory"/> managed by the given /// writer. /// </summary> /// <exception cref="IOException"></exception> public IndexAndTaxonomyRevision(IndexWriter indexWriter, SnapshotDirectoryTaxonomyWriter taxonomyWriter) { this.indexSdp = indexWriter.Config.IndexDeletionPolicy as SnapshotDeletionPolicy; if (indexSdp == null) { throw new ArgumentException("IndexWriter must be created with SnapshotDeletionPolicy", "indexWriter"); } this.indexWriter = indexWriter; this.taxonomyWriter = taxonomyWriter; this.taxonomySdp = taxonomyWriter.DeletionPolicy; this.indexCommit = indexSdp.Snapshot(); this.taxonomyCommit = taxonomySdp.Snapshot(); this.version = RevisionVersion(indexCommit, taxonomyCommit); this.sourceFiles = RevisionFiles(indexCommit, taxonomyCommit); }
public override void Run() { try { Iw.Commit(); IndexCommit ic = Sdp.Snapshot(); foreach (string name in ic.FileNames) { //distribute, and backup //System.out.println(names); Assert.IsTrue(SlowFileExists(Dir, name)); } } catch (Exception e) { throw new Exception(e.Message, e); } }
public override void Run() { try { iw.Commit(); IndexCommit ic = sdp.Snapshot(); foreach (string name in ic.FileNames) { //distribute, and backup //System.out.println(names); assertTrue(SlowFileExists(dir, name)); } } catch (Exception e) { throw new Exception(e.toString(), e); } }
public async Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter) { Guard.NotNull(directory); Guard.NotNull(snapshotter); var directoryInfo = ((FSDirectory)directory).Directory; var commit = snapshotter.Snapshot(); try { using (var fileStream = GetArchiveStream(directoryInfo)) { using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true)) { foreach (var fileName in commit.FileNames) { var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName)); try { if (!file.Name.Equals(ArchiveFile, StringComparison.OrdinalIgnoreCase) && !file.Name.Equals(LockFile, StringComparison.OrdinalIgnoreCase)) { zipArchive.CreateEntryFromFile(file.FullName, file.Name); } } catch (IOException) { continue; } } } fileStream.Position = 0; await assetStore.UploadAsync(directoryInfo.Name, fileStream, true); } } finally { snapshotter.Release(commit); } }
public async Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter) { var directoryInfo = ((FSDirectory)directory).Directory; var commit = snapshotter.Snapshot(); try { var fileId = directoryInfo.Name; try { await bucket.DeleteAsync(fileId); } catch (GridFSFileNotFoundException) { } using (var stream = await bucket.OpenUploadStreamAsync(fileId, fileId)) { using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, true)) { foreach (var fileName in commit.FileNames) { var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName)); using (var fileStream = file.OpenRead()) { var entry = zipArchive.CreateEntry(fileStream.Name); using (var entryStream = entry.Open()) { await fileStream.CopyToAsync(entryStream); } } } } } } finally { snapshotter.Release(commit); } }
public async Task FlushAsync() { if (updates > 0 && indexWriter != null) { indexWriter.Commit(); indexWriter.Flush(true, true); CleanReader(); var commit = snapshotter.Snapshot(); try { await assetStore.UploadDirectoryAsync(directory, commit); } finally { snapshotter.Release(commit); } updates = 0; } }
public void Backup(string backupDirectory) { bool hasSnapshot = false; bool throwOnFinallyException = true; try { var existingFiles = new HashSet <string>(); var allFilesPath = Path.Combine(backupDirectory, "index-files.all-existing-index-files"); var saveToFolder = Path.Combine(backupDirectory, "Indexes"); Directory.CreateDirectory(saveToFolder); if (File.Exists(allFilesPath)) { foreach (var file in File.ReadLines(allFilesPath)) { existingFiles.Add(file); } } var neededFilePath = Path.Combine(saveToFolder, "index-files.required-for-index-restore"); using (var allFilesWriter = File.Exists(allFilesPath) ? File.AppendText(allFilesPath) : File.CreateText(allFilesPath)) using (var neededFilesWriter = File.CreateText(neededFilePath)) { var segmentsFileName = "segments.gen"; var segmentsFullPath = Path.Combine(indexDirectory, segmentsFileName); File.Copy(segmentsFullPath, Path.Combine(saveToFolder, segmentsFileName)); allFilesWriter.WriteLine(segmentsFileName); neededFilesWriter.WriteLine(segmentsFileName); var versionFileName = "index.version"; var versionFullPath = Path.Combine(indexDirectory, versionFileName); File.Copy(versionFullPath, Path.Combine(saveToFolder, versionFileName)); allFilesWriter.WriteLine(versionFileName); neededFilesWriter.WriteLine(versionFileName); var commit = snapshotter.Snapshot(); hasSnapshot = true; foreach (var fileName in commit.FileNames) { var fullPath = Path.Combine(indexDirectory, fileName); if (".lock".Equals(Path.GetExtension(fullPath), StringComparison.InvariantCultureIgnoreCase)) { continue; } if (File.Exists(fullPath) == false) { continue; } if (existingFiles.Contains(fileName) == false) { var destFileName = Path.Combine(saveToFolder, fileName); try { File.Copy(fullPath, destFileName); } catch (Exception e) { Log.WarnException( "Could not backup RavenFS index" + " because failed to copy file : " + fullPath + ". Skipping the index, will force index reset on restore", e); //TODO: is it also true for RavenFS? neededFilesWriter.Dispose(); TryDelete(neededFilePath); return; } allFilesWriter.WriteLine(fileName); } neededFilesWriter.WriteLine(fileName); } allFilesWriter.Flush(); neededFilesWriter.Flush(); } } catch { throwOnFinallyException = false; throw; } finally { if (snapshotter != null && hasSnapshot) { try { snapshotter.Release(); } catch { if (throwOnFinallyException) { throw; } } } } }
public bool Copy(SimpleFSDirectory sourceLuceneDirectory, Analyzer analyzer, DirectoryInfo targetPath) { if (targetPath.Exists == false) { Directory.CreateDirectory(targetPath.FullName); } //copy index if it exists, don't do anything if it's not there if (IndexReader.IndexExists(sourceLuceneDirectory) == false) { return(true); } using (new IndexWriter( //read from the underlying/default directory, not the temp codegen dir sourceLuceneDirectory, analyzer, _snapshotter, IndexWriter.MaxFieldLength.UNLIMITED)) { try { //var basePath = IOHelper.MapPath(configuredPath); var commit = _snapshotter.Snapshot(); var allSnapshotFiles = commit.GetFileNames() .Concat(new[] { commit.GetSegmentsFileName(), //we need to manually include the segments.gen file "segments.gen" }) .Distinct() .ToArray(); //Get all files in the temp storage that don't exist in the snapshot collection, we want to remove these var toRemove = targetPath.GetFiles() .Select(x => x.Name) .Except(allSnapshotFiles); //using (var tempDirectory = new SimpleFSDirectory(tempDir)) //{ //TODO: We're ignoring if it is locked right now, it shouldn't be unless for some strange reason the // last process hasn't fully shut down, in that case we're not going to worry about it. //if (IndexWriter.IsLocked(tempDirectory) == false) //{ foreach (var file in toRemove) { try { File.Delete(Path.Combine(targetPath.FullName, file)); } catch (IOException ex) { //TODO: we're ignoring this, as old files shouldn't affect the index/search operations, lucene files are 'write once' //TODO: Return some info? //quit here //return false; } } //} //else //{ // LogHelper.Warn<LocalTempStorageIndexer>("Cannot sync index files from main storage, the index is currently locked"); // //quit here // return false; //} foreach (var fileName in allSnapshotFiles.Where(f => string.IsNullOrWhiteSpace(f) == false)) { var destination = Path.Combine(targetPath.FullName, Path.GetFileName(fileName)); //don't copy if it's already there, lucene is 'write once' so this file is meant to be there already if (File.Exists(destination)) { continue; } try { File.Copy( Path.Combine(sourceLuceneDirectory.GetDirectory().FullName, "Index", fileName), destination); } catch (IOException ex) { //TODO: Return some info? //quit here return(false); } } //} } finally { _snapshotter.Release(); } } return(true); }