public AzureIndexInput(AzureDirectory azureDirectory, string name, BlobClient blob) : base(name) { this._name = name; this._azureDirectory = azureDirectory; #if FULLDEBUG Debug.WriteLine($"{_azureDirectory.Name} opening {name} "); #endif _fileMutex = BlobMutexManager.GrabMutex(name); _fileMutex.WaitOne(); try { _blobContainer = azureDirectory.BlobContainer; _blob = blob; bool fileNeeded = false; if (!CacheDirectory.FileExists(name)) { fileNeeded = true; } else { long cachedLength = CacheDirectory.FileLength(name); var properties = blob.GetProperties(); long blobLength = properties.Value?.ContentLength ?? 0; if (cachedLength != blobLength) { fileNeeded = true; } } // if the file does not exist // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage) if (fileNeeded) { using (StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(name)) { // get the blob _blob.DownloadTo(fileStream); fileStream.Flush(); Debug.WriteLine($"{_azureDirectory.Name} GET {_name} RETREIVED {fileStream.Length} bytes"); } } #if FULLDEBUG Debug.WriteLine($"{_azureDirectory.Name} Using cached file for {name}"); #endif // and open it as our input, this is now available forevers until new file comes along _indexInput = CacheDirectory.OpenInput(name, IOContext.DEFAULT); } finally { _fileMutex.ReleaseMutex(); } }
public string ReadCachedFileETag(string name) { var fileName = name + ".etag"; if (CacheDirectory.FileExists(name) == false) { return(String.Empty); } using (var input = CacheDirectory.OpenInput(fileName)) { return(input.ReadString()); } }
public override void Close() { _fileMutex.WaitOne(); try { string fileName = _name; // make sure it's all written out if (_indexOutput != null) { _indexOutput.Flush(); _indexOutput.Close(); } if (CacheDirectory.FileExists(fileName)) { //open stream to read cache file using (var cacheStream = new StreamInput(CacheDirectory.OpenInput(fileName))) // push the blobStream up to the master using (var masterStream = new StreamOutput(MasterDirectory.CreateOutput(fileName))) { cacheStream.CopyTo(masterStream); masterStream.Flush(); Trace.WriteLine(string.Format("PUT {1} bytes to {0} in cloud", _name, cacheStream.Length)); } //sync the last file write times - at least get them close. //TODO: The alternative would be to force both directory instances to be FSDirectory, // or try casting the master directory to FSDirectory to get the raw FileInfo and manually // set the lastmodified time - this should work though MasterDirectory.TouchFile(fileName); CacheDirectory.TouchFile(fileName); #if FULLDEBUG Debug.WriteLine(string.Format("CLOSED WRITESTREAM {0}", _name)); #endif } // clean up _indexOutput = null; GC.SuppressFinalize(this); } finally { _fileMutex.ReleaseMutex(); } }
/// <summary> /// Removes an existing file in the directory. /// </summary> /// <param name="name">The name.</param> public override void DeleteFile(string name) { lock (m_syncRoot) { var fileUrl = SPUtility.ConcatUrls(m_folder.Value.ServerRelativeUrl, name); var file = m_web.GetFile(fileUrl); file.Delete(); } if (CacheDirectory.FileExists(name)) { CacheDirectory.DeleteFile(name); } if (CacheDirectory.FileExists(name + ".etag")) { CacheDirectory.DeleteFile(name + ".etag"); } }
public CompositeIndexInput(SyncDirectory azuredirectory, string name, string resourceDescription) : base(resourceDescription) { _name = name; #if FULLDEBUG Debug.WriteLine(String.Format("opening {0} ", _name)); #endif _fileMutex = SyncMutexManager.GrabMutex(_name); _fileMutex.WaitOne(); try { _syncDirectory = azuredirectory; var fileName = _name; var fFileNeeded = false; if (!CacheDirectory.FileExists(fileName)) { fFileNeeded = true; } else { long cachedLength = CacheDirectory.FileLength(fileName); long primaryLength = PrimaryDirectory.FileLength(fileName); if (cachedLength != primaryLength) { fFileNeeded = true; } else { // cachedLastModifiedUTC was not ouputting with a date (just time) and the time was always off var cachedFilePath = Path.Combine(_syncDirectory.CacheDirectoryPath, fileName); var primaryFilePath = Path.Combine(_syncDirectory.PrimaryDirectoryPath, fileName); var cachedLastModified = File.GetLastWriteTimeUtc(cachedFilePath); var primaryLastModified = File.GetLastWriteTimeUtc(primaryFilePath); if (cachedLastModified != primaryLastModified) { var timeSpan = primaryLastModified.Subtract(cachedLastModified); if (timeSpan.TotalSeconds > 1) { fFileNeeded = true; } else { #if FULLDEBUG Debug.WriteLine(timeSpan.TotalSeconds); #endif // file not needed } } } } // if the file does not exist // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage) if (fFileNeeded) { var primaryInput = PrimaryDirectory.OpenInput(fileName, IOContext.DEFAULT); byte[] primaryInputBytes = new byte[primaryInput.Length]; primaryInput.ReadBytes(primaryInputBytes, 0, (int)primaryInputBytes.Length); using (var cachedOutput = CacheDirectory.CreateOutput(fileName, IOContext.DEFAULT)) { cachedOutput.WriteBytes(primaryInputBytes, (int)primaryInputBytes.Length); cachedOutput.Flush(); } primaryInput.Dispose(); var cachedFilePath = Path.Combine(_syncDirectory.CacheDirectoryPath, fileName); var primaryFilePath = Path.Combine(_syncDirectory.PrimaryDirectoryPath, fileName); var primaryLastModified = File.GetLastWriteTimeUtc(primaryFilePath); File.SetLastWriteTimeUtc(cachedFilePath, primaryLastModified); //using (var fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName)) //{ // // get the blob // _blob.DownloadToStream(fileStream); // fileStream.Flush(); // Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length)); //} // and open it as an input _indexInput = CacheDirectory.OpenInput(fileName, IOContext.DEFAULT); } else { #if FULLDEBUG Debug.WriteLine(String.Format("Using cached file for {0}", _name)); #endif // open the file in read only mode _indexInput = CacheDirectory.OpenInput(fileName, IOContext.DEFAULT); } } finally { _fileMutex.ReleaseMutex(); } }
public SyncIndexInput(SyncDirectory directory, string name) { if (directory == null) { throw new ArgumentNullException(nameof(directory)); } _name = name; _syncDirectory = directory; #if FULLDEBUG Trace.WriteLine(String.Format("opening {0} ", _name)); #endif _fileMutex = SyncMutexManager.GrabMutex(_syncDirectory, _name); _fileMutex.WaitOne(); try { var fileName = _name; var fFileNeeded = false; if (!CacheDirectory.FileExists(fileName)) { fFileNeeded = true; } else { long cachedLength = CacheDirectory.FileLength(fileName); long masterLength = MasterDirectory.FileLength(fileName); if (cachedLength != masterLength) { fFileNeeded = true; } else { long cacheDate = CacheDirectory.FileModified(fileName); long masterDate = MasterDirectory.FileModified(fileName); //we need to compare to the second instead of by ticks because when we call //TouchFile in SyncIndexOutput this won't set the files to be identical DateTime start = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); var cachedLastModifiedUTC = start.AddMilliseconds(cacheDate).ToUniversalTime(); var masterLastModifiedUTC = start.AddMilliseconds(masterDate).ToUniversalTime(); if (cachedLastModifiedUTC != masterLastModifiedUTC) { var timeSpan = masterLastModifiedUTC.Subtract(cachedLastModifiedUTC); //NOTE: This heavily depends on TouchFile in SyncIndexOutput which sets both the //master and slave files to be 'Now', in theory this operation shouldn't //make the file times any bigger than 1 second if (timeSpan.TotalSeconds > 2) { fFileNeeded = true; } else { #if FULLDEBUG Debug.WriteLine(timeSpan.TotalSeconds); #endif // file not needed } } } } // if the file does not exist // or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage) if (fFileNeeded) { //get the master file stream using (var masterStream = new StreamInput(MasterDirectory.OpenInput(fileName))) using (var cacheStream = _syncDirectory.CreateCachedOutputAsStream(fileName)) { //copy this to the cached file stream masterStream.CopyTo(cacheStream); cacheStream.Flush(); Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, cacheStream.Length)); } // and open it as an input _indexInput = CacheDirectory.OpenInput(fileName); } else { #if FULLDEBUG Debug.WriteLine(String.Format("Using cached file for {0}", _name)); #endif // open the file in read only mode _indexInput = CacheDirectory.OpenInput(fileName); } } finally { _fileMutex.ReleaseMutex(); } }