public HttpResponseMessage Get(string name) { name = FileHeader.Canonize(name); FileAndPagesInformation fileAndPages = null; Storage.Batch(accessor => fileAndPages = accessor.GetFile(name, 0, 0)); if (fileAndPages.Metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker)) { if (log.IsDebugEnabled) { log.Debug("File '{0}' is not accessible to get (Raven-Delete-Marker set)", name); } throw new HttpResponseException(HttpStatusCode.NotFound); } var readingStream = StorageStream.Reading(Storage, name); var filename = GetFileName(name, fileAndPages.Metadata); var result = StreamResult(filename, readingStream); var etag = new Etag(fileAndPages.Metadata.Value <string>(Constants.MetadataEtagField)); fileAndPages.Metadata.Remove(Constants.MetadataEtagField); WriteHeaders(fileAndPages.Metadata, etag, result); if (log.IsDebugEnabled) { log.Debug("File '{0}' with etag {1} is being retrieved.", name, etag); } return(result.WithNoCache()); }
public HttpResponseMessage Get(string name) { name = RavenFileNameHelper.RavenPath(name); FileAndPagesInformation fileAndPages = null; try { Storage.Batch(accessor => fileAndPages = accessor.GetFile(name, 0, 0)); } catch (FileNotFoundException) { log.Debug("File '{0}' was not found", name); throw new HttpResponseException(HttpStatusCode.NotFound); } if (fileAndPages.Metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker)) { log.Debug("File '{0}' is not accessible to get (Raven-Delete-Marker set)", name); throw new HttpResponseException(HttpStatusCode.NotFound); } var readingStream = StorageStream.Reading(Storage, name); var result = StreamResult(name, readingStream); var etag = new Etag(fileAndPages.Metadata.Value <string>(Constants.MetadataEtagField)); fileAndPages.Metadata.Remove(Constants.MetadataEtagField); WriteHeaders(fileAndPages.Metadata, etag, result); return(result.WithNoCache()); }
public Task <Stream> DownloadFile(FileHeader file) { var name = file.FullPath; var readingStream = StorageStream.Reading(filesystem.Storage, name); return(new CompletedTask <Stream>(readingStream)); }
private void StreamExportToClient(Stream gzip2, string[] fileNames) { using (var gzip = new BufferedStream(gzip2)) // using (var gzip = new GZipStream(stream, CompressionMode.Compress,true)) { var binaryWriter = new BinaryWriter(gzip); var buffer = new byte[StorageConstants.MaxPageSize]; var pageBuffer = new byte[StorageConstants.MaxPageSize]; foreach (var name in fileNames) { FileAndPagesInformation fileAndPages = null; var cannonizedName = FileHeader.Canonize(name); try { Storage.Batch(accessor => fileAndPages = accessor.GetFile(cannonizedName, 0, 0)); } catch (Exception ex) { throw; } // if we didn't find the document, we'll write "-1" to the stream, signaling that if (fileAndPages.Metadata.Keys.Contains(SynchronizationConstants.RavenDeleteMarker)) { if (log.IsDebugEnabled) { log.Debug("File '{0}' is not accessible to get (Raven-Delete-Marker set)", name); } binaryWriter.Write(-1); continue; } var fileSize = fileAndPages.UploadedSize; binaryWriter.Write(fileSize); var readingStream = StorageStream.Reading(Storage, cannonizedName); var bytesRead = 0; do { try { bytesRead = readingStream.ReadUsingExternalTempBuffer(buffer, 0, buffer.Length, pageBuffer); } catch (Exception ex) { throw; } gzip.Write(buffer, 0, bytesRead); } while (bytesRead > 0); } } //gzip2.Flush(); }
public async Task <SynchronizationReport> UploadToAsync(IAsyncFilesSynchronizationCommands destination) { using (var sourceFileStream = StorageStream.Reading(Storage, FileName)) { var fileSize = sourceFileStream.Length; var onlySourceNeed = new List <RdcNeed> { new RdcNeed { BlockType = RdcNeedType.Source, BlockLength = (ulong)fileSize, FileOffset = 0 } }; return(await PushByUsingMultipartRequest(destination, sourceFileStream, onlySourceNeed)); } }
public async Task <SynchronizationReport> UploadToAsync(ISynchronizationServerClient synchronizationServerClient) { using (var sourceFileStream = StorageStream.Reading(Storage, FileName)) { var fileSize = sourceFileStream.Length; var onlySourceNeed = new List <RdcNeed> { new RdcNeed { BlockType = RdcNeedType.Source, BlockLength = (ulong)fileSize, FileOffset = 0 } }; return(await PushByUsingMultipartRequest(synchronizationServerClient, sourceFileStream, onlySourceNeed).ConfigureAwait(false)); } }
private async Task <SynchronizationReport> SynchronizeTo(IAsyncFilesSynchronizationCommands destination, ISignatureRepository localSignatureRepository, ISignatureRepository remoteSignatureRepository, SignatureManifest sourceSignatureManifest, SignatureManifest destinationSignatureManifest) { var seedSignatureInfo = SignatureInfo.Parse(destinationSignatureManifest.Signatures.Last().Name); var sourceSignatureInfo = SignatureInfo.Parse(sourceSignatureManifest.Signatures.Last().Name); using (var localFile = StorageStream.Reading(Storage, FileName)) { IList <RdcNeed> needList; using (var needListGenerator = new NeedListGenerator(remoteSignatureRepository, localSignatureRepository)) { needList = needListGenerator.CreateNeedsList(seedSignatureInfo, sourceSignatureInfo, Cts.Token); } return(await PushByUsingMultipartRequest(destination, localFile, needList)); } }
public void StorageStream_can_read_overlaping_byte_ranges_from_last_page() { var buffer = new byte[StorageConstants.MaxPageSize]; new Random().NextBytes(buffer); using (var stream = StorageStream.CreatingNewAndWritting( transactionalStorage, new MockIndexStorage(), new StorageOperationsTask(transactionalStorage, new MockIndexStorage(), new EmptyNotificationsPublisher()), "file", EmptyETagMetadata)) { stream.Write(buffer, 0, StorageConstants.MaxPageSize); } using (var stream = StorageStream.Reading(transactionalStorage, "file")) { var readBuffer = new byte[10]; stream.Seek(StorageConstants.MaxPageSize - 10, SeekOrigin.Begin); stream.Read(readBuffer, 0, 10); // read last 10 bytes var subBuffer = buffer.ToList().Skip(StorageConstants.MaxPageSize - 10).Take(10).ToArray(); for (int i = 0; i < 10; i++) { Assert.Equal(subBuffer[i], readBuffer[i]); } readBuffer = new byte[5]; stream.Seek(StorageConstants.MaxPageSize - 5, SeekOrigin.Begin); stream.Read(readBuffer, 0, 5); // read last 5 bytes - note that they were read last time as well subBuffer = buffer.ToList().Skip(StorageConstants.MaxPageSize - 5).Take(5).ToArray(); for (int i = 0; i < 5; i++) { Assert.Equal(subBuffer[i], readBuffer[i]); } } }
public void StorageStream_can_read_overlaping_byte_ranges_from_last_page() { var buffer = new byte[StorageConstants.MaxPageSize]; new Random().NextBytes(buffer); using (var stream = StorageStream.CreatingNewAndWritting(fs, "file", new RavenJObject())) { stream.Write(buffer, 0, StorageConstants.MaxPageSize); } using (var stream = StorageStream.Reading(fs.Storage, "file")) { var readBuffer = new byte[10]; stream.Seek(StorageConstants.MaxPageSize - 10, SeekOrigin.Begin); stream.Read(readBuffer, 0, 10); // read last 10 bytes var subBuffer = buffer.ToList().Skip(StorageConstants.MaxPageSize - 10).Take(10).ToArray(); for (int i = 0; i < 10; i++) { Assert.Equal(subBuffer[i], readBuffer[i]); } readBuffer = new byte[5]; stream.Seek(StorageConstants.MaxPageSize - 5, SeekOrigin.Begin); stream.Read(readBuffer, 0, 5); // read last 5 bytes - note that they were read last time as well subBuffer = buffer.ToList().Skip(StorageConstants.MaxPageSize - 5).Take(5).ToArray(); for (int i = 0; i < 5; i++) { Assert.Equal(subBuffer[i], readBuffer[i]); } } }
public async Task <HttpResponseMessage> MultipartProceed(string fileSystemName) { if (!Request.Content.IsMimeMultipartContent()) { throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType); } var fileName = Request.Headers.GetValues(SyncingMultipartConstants.FileName).FirstOrDefault(); var canonicalFilename = FileHeader.Canonize(fileName); var tempFileName = RavenFileNameHelper.DownloadingFileName(canonicalFilename); var sourceServerInfo = ReadInnerHeaders.Value <ServerInfo>(SyncingMultipartConstants.SourceServerInfo); var sourceFileETag = Guid.Parse(GetHeader(Constants.MetadataEtagField).Trim('\"')); var report = new SynchronizationReport(canonicalFilename, sourceFileETag, SynchronizationType.ContentUpdate); Log.Debug("Starting to process multipart synchronization request of a file '{0}' with ETag {1} from {2}", fileName, sourceFileETag, sourceServerInfo); StorageStream localFile = null; var isNewFile = false; var isConflictResolved = false; try { Storage.Batch(accessor => { AssertFileIsNotBeingSynced(canonicalFilename, accessor); FileLockManager.LockByCreatingSyncConfiguration(canonicalFilename, sourceServerInfo, accessor); }); SynchronizationTask.IncomingSynchronizationStarted(canonicalFilename, sourceServerInfo, sourceFileETag, SynchronizationType.ContentUpdate); PublishSynchronizationNotification(fileSystemName, canonicalFilename, sourceServerInfo, report.Type, SynchronizationAction.Start); Storage.Batch(accessor => StartupProceed(canonicalFilename, accessor)); RavenJObject sourceMetadata = GetFilteredMetadataFromHeaders(ReadInnerHeaders); var localMetadata = GetLocalMetadata(canonicalFilename); if (localMetadata != null) { AssertConflictDetection(canonicalFilename, localMetadata, sourceMetadata, sourceServerInfo, out isConflictResolved); localFile = StorageStream.Reading(Storage, canonicalFilename); } else { isNewFile = true; } Historian.UpdateLastModified(sourceMetadata); var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWriting(Storage, Search, StorageOperationsTask, tempFileName, sourceMetadata); var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile); Log.Debug("Starting to process/read multipart content of a file '{0}'", fileName); await Request.Content.ReadAsMultipartAsync(provider); Log.Debug("Multipart content of a file '{0}' was processed/read", fileName); report.BytesCopied = provider.BytesCopied; report.BytesTransfered = provider.BytesTransfered; report.NeedListLength = provider.NumberOfFileParts; synchronizingFile.PreventUploadComplete = false; synchronizingFile.Flush(); synchronizingFile.Dispose(); sourceMetadata["Content-MD5"] = synchronizingFile.FileHash; Storage.Batch(accessor => accessor.UpdateFileMetadata(tempFileName, sourceMetadata)); Storage.Batch(accessor => { StorageOperationsTask.IndicateFileToDelete(canonicalFilename); accessor.RenameFile(tempFileName, canonicalFilename); Search.Delete(tempFileName); Search.Index(canonicalFilename, sourceMetadata); }); if (isNewFile) { Log.Debug("Temporary downloading file '{0}' was renamed to '{1}'. Indexes were updated.", tempFileName, fileName); } else { Log.Debug("Old file '{0}' was deleted. Indexes were updated.", fileName); } if (isConflictResolved) { ConflictArtifactManager.Delete(canonicalFilename); } } catch (Exception ex) { if (ShouldAddExceptionToReport(ex)) { report.Exception = ex; } } finally { if (localFile != null) { localFile.Dispose(); } } if (report.Exception == null) { Log.Debug( "File '{0}' was synchronized successfully from {1}. {2} bytes were transfered and {3} bytes copied. Need list length was {4}", fileName, sourceServerInfo, report.BytesTransfered, report.BytesCopied, report.NeedListLength); } else { Log.WarnException( string.Format("Error has occurred during synchronization of a file '{0}' from {1}", fileName, sourceServerInfo), report.Exception); } FinishSynchronization(canonicalFilename, report, sourceServerInfo, sourceFileETag); PublishFileNotification(fileName, isNewFile ? FileChangeAction.Add : FileChangeAction.Update); PublishSynchronizationNotification(fileSystemName, fileName, sourceServerInfo, report.Type, SynchronizationAction.Finish); if (isConflictResolved) { Publisher.Publish(new ConflictNotification { FileName = fileName, Status = ConflictStatus.Resolved }); } return(GetMessageWithObject(report)); }
private async Task ExecuteContentUpdate(RavenJObject localMetadata, SynchronizationReport report) { var tempFileName = RavenFileNameHelper.DownloadingFileName(fileName); using (var localFile = localMetadata != null ? StorageStream.Reading(fs.Storage, fileName) : null) { fs.PutTriggers.Apply(trigger => trigger.OnPut(tempFileName, sourceMetadata)); fs.Historian.UpdateLastModified(sourceMetadata); var synchronizingFile = SynchronizingFileStream.CreatingOrOpeningAndWriting(fs, tempFileName, sourceMetadata); fs.PutTriggers.Apply(trigger => trigger.AfterPut(tempFileName, null, sourceMetadata)); var provider = new MultipartSyncStreamProvider(synchronizingFile, localFile); if (Log.IsDebugEnabled) { Log.Debug("Starting to process/read multipart content of a file '{0}'", fileName); } await MultipartContent.ReadAsMultipartAsync(provider).ConfigureAwait(false); if (Log.IsDebugEnabled) { Log.Debug("Multipart content of a file '{0}' was processed/read", fileName); } report.BytesCopied = provider.BytesCopied; report.BytesTransfered = provider.BytesTransfered; report.NeedListLength = provider.NumberOfFileParts; synchronizingFile.PreventUploadComplete = false; synchronizingFile.Flush(); synchronizingFile.Dispose(); sourceMetadata["Content-MD5"] = synchronizingFile.FileHash; FileUpdateResult updateResult = null; fs.Storage.Batch(accessor => updateResult = accessor.UpdateFileMetadata(tempFileName, sourceMetadata, null)); fs.Storage.Batch(accessor => { using (fs.DisableAllTriggersForCurrentThread()) { fs.Files.IndicateFileToDelete(fileName, null); } accessor.RenameFile(tempFileName, fileName); fs.Search.Delete(tempFileName); fs.Search.Index(fileName, sourceMetadata, updateResult.Etag); }); if (Log.IsDebugEnabled) { var message = localFile == null ? string.Format("Temporary downloading file '{0}' was renamed to '{1}'. Indexes were updated.", tempFileName, fileName) : string.Format("Old file '{0}' was deleted. Indexes were updated.", fileName); Log.Debug(message); } fs.Publisher.Publish(new FileChangeNotification { File = fileName, Action = localFile == null ? FileChangeAction.Add : FileChangeAction.Update }); } }
private IEnumerable <SignatureInfo> PrepareSignatures(string filename) { var input = StorageStream.Reading(_transactionalStorage, filename); return(_sigGenerator.GenerateSignatures(input, filename, _signatureRepository)); }
private void WriteFilesAsAttachments(JsonTextWriter jsonWriter) { int totalFilesCount = 0; fileStorage.Batch(accsesor => { totalFilesCount = accsesor.GetFileCount(); }); if (totalFilesCount == 0) { return; } int currentFilesCount = 0; int previousFilesCount = 0; var lastEtag = Etag.Empty; do { try { previousFilesCount = currentFilesCount; fileStorage.Batch(accsesor => { var fileHeaders = accsesor.GetFilesAfter(lastEtag, batchSize); foreach (var header in fileHeaders) { var file = StorageStream.Reading(fileStorage, header.FullPath); jsonWriter.WriteStartObject(); jsonWriter.WritePropertyName("Data"); int read = file.Read(filesBuffer, 0, filesBuffer.Length); jsonWriter.WriteRaw("\""); while (read > 0) { var base64 = Convert.ToBase64String(filesBuffer, 0, read); jsonWriter.WriteRaw(base64); jsonWriter.Flush(); read = file.Read(filesBuffer, 0, filesBuffer.Length); } jsonWriter.WriteRaw("\""); jsonWriter.SealValue(); jsonWriter.WritePropertyName("Metadata"); fileMetadata.WriteTo(jsonWriter); jsonWriter.WritePropertyName("Key"); jsonWriter.WriteValue(header.FullPath); jsonWriter.WritePropertyName("Etag"); jsonWriter.WriteValue(header.Etag.ToString()); jsonWriter.WriteEndObject(); lastEtag = header.Etag; currentFilesCount++; if (currentFilesCount % batchSize == 0) { ReportProgress("files", currentFilesCount, totalFilesCount); } } }); } catch (Exception e) { lastEtag = lastEtag.IncrementBy(1); currentFilesCount++; ReportCorrupted("files", currentFilesCount, e.Message); } finally { if (currentFilesCount > previousFilesCount) { ReportProgress("files", currentFilesCount, totalFilesCount); } } } while (currentFilesCount < totalFilesCount); }
public async Task eventually_will_cleanup_all_deleted_files_even_if_they_use_the_same_pages_and_concurrency_exceptions_are_thrown(string storage) { var client = NewAsyncClient(requestedStorage: storage); var rfs = GetFileSystem(); var bytes1 = new byte[1024 * 1024 * 3]; var bytes2 = new byte[1024 * 1024 * 2]; var random = new Random(); random.NextBytes(bytes1); random.NextBytes(bytes2); await client.UploadAsync("1.bin", new MemoryStream(bytes1)); await client.UploadAsync("1.bin", new MemoryStream(bytes2)); // will indicate old file to delete await client.UploadAsync("1.bin", new MemoryStream(bytes1)); // will indicate old file to delete await client.UploadAsync("1.bin", new MemoryStream(bytes2)); // will indicate old file to delete await client.UploadAsync("2.bin", new MemoryStream(bytes2)); await client.UploadAsync("2.bin", new MemoryStream(bytes1)); // will indicate old file to delete await client.UploadAsync("2.bin", new MemoryStream(bytes2)); // will indicate old file to delete await client.DeleteAsync("1.bin"); await client.DeleteAsync("2.bin"); var sw = Stopwatch.StartNew(); while (sw.Elapsed < TimeSpan.FromMinutes(1)) { try { await rfs.Files.CleanupDeletedFilesAsync(); break; } catch (Exception e) { if (e.InnerException is ConcurrencyException == false) { throw; } // concurrency exceptions are expected because files indicated to deletion use the same pages // so concurrent modifications can result in concurrency exception // however the first deletion attempt should pass, so finally all deleted files should be cleaned up } } IList <FileHeader> files = null; rfs.Storage.Batch(accessor => { files = accessor.GetFilesAfter(Etag.Empty, 10).ToList(); }); Assert.Equal(2, files.Count); Assert.True(files.All(x => x.IsTombstone)); // but after upload there should be two files which aren't tombstones await client.UploadAsync("1.bin", new MemoryStream(bytes1)); await client.UploadAsync("2.bin", new MemoryStream(bytes1)); rfs.Storage.Batch(accessor => { files = accessor.GetFilesAfter(Etag.Empty, 10).ToList(); }); Assert.Equal(2, files.Count); Assert.False(files.All(x => x.IsTombstone)); foreach (var file in files) { var content = new MemoryStream(new byte[file.TotalSize.Value]); await StorageStream.Reading(rfs.Storage, files[0].FullPath).CopyToAsync(content); Assert.Equal(bytes1, content.ToArray()); } }