public AzureIndexInput(AzureDirectory azureDirectory, string name, BlobClient blob)
: base(name)
{
this._name = name;
this._azureDirectory = azureDirectory;
#if FULLDEBUG
Debug.WriteLine($"{_azureDirectory.Name} opening {name} ");
#endif
_fileMutex = BlobMutexManager.GrabMutex(name);
_fileMutex.WaitOne();
try
{
_blobContainer = azureDirectory.BlobContainer;
_blob = blob;
bool fileNeeded = false;
if (!CacheDirectory.FileExists(name))
{
fileNeeded = true;
}
else
{
long cachedLength = CacheDirectory.FileLength(name);
var properties = blob.GetProperties();
long blobLength = properties.Value?.ContentLength ?? 0;
if (cachedLength != blobLength)
{
fileNeeded = true;
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if (fileNeeded)
{
using (StreamOutput fileStream = _azureDirectory.CreateCachedOutputAsStream(name))
{
// get the blob
_blob.DownloadTo(fileStream);
fileStream.Flush();
Debug.WriteLine($"{_azureDirectory.Name} GET {_name} RETREIVED {fileStream.Length} bytes");
}
}
#if FULLDEBUG
Debug.WriteLine($"{_azureDirectory.Name} Using cached file for {name}");
#endif
// and open it as our input, this is now available forevers until new file comes along
_indexInput = CacheDirectory.OpenInput(name, IOContext.DEFAULT);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
public string ReadCachedFileETag(string name)
{
var fileName = name + ".etag";
if (CacheDirectory.FileExists(name) == false)
{
return(String.Empty);
}
using (var input = CacheDirectory.OpenInput(fileName))
{
return(input.ReadString());
}
}
public override void Close()
{
_fileMutex.WaitOne();
try
{
string fileName = _name;
// make sure it's all written out
if (_indexOutput != null)
{
_indexOutput.Flush();
_indexOutput.Close();
}
if (CacheDirectory.FileExists(fileName))
{
//open stream to read cache file
using (var cacheStream = new StreamInput(CacheDirectory.OpenInput(fileName)))
// push the blobStream up to the master
using (var masterStream = new StreamOutput(MasterDirectory.CreateOutput(fileName)))
{
cacheStream.CopyTo(masterStream);
masterStream.Flush();
Trace.WriteLine(string.Format("PUT {1} bytes to {0} in cloud", _name, cacheStream.Length));
}
//sync the last file write times - at least get them close.
//TODO: The alternative would be to force both directory instances to be FSDirectory,
// or try casting the master directory to FSDirectory to get the raw FileInfo and manually
// set the lastmodified time - this should work though
MasterDirectory.TouchFile(fileName);
CacheDirectory.TouchFile(fileName);
#if FULLDEBUG
Debug.WriteLine(string.Format("CLOSED WRITESTREAM {0}", _name));
#endif
}
// clean up
_indexOutput = null;
GC.SuppressFinalize(this);
}
finally
{
_fileMutex.ReleaseMutex();
}
}
/// <summary>
/// Removes an existing file in the directory.
/// </summary>
/// <param name="name">The name.</param>
public override void DeleteFile(string name)
{
lock (m_syncRoot)
{
var fileUrl = SPUtility.ConcatUrls(m_folder.Value.ServerRelativeUrl, name);
var file = m_web.GetFile(fileUrl);
file.Delete();
}
if (CacheDirectory.FileExists(name))
{
CacheDirectory.DeleteFile(name);
}
if (CacheDirectory.FileExists(name + ".etag"))
{
CacheDirectory.DeleteFile(name + ".etag");
}
}
public CompositeIndexInput(SyncDirectory azuredirectory, string name, string resourceDescription) : base(resourceDescription)
{
_name = name;
#if FULLDEBUG
Debug.WriteLine(String.Format("opening {0} ", _name));
#endif
_fileMutex = SyncMutexManager.GrabMutex(_name);
_fileMutex.WaitOne();
try
{
_syncDirectory = azuredirectory;
var fileName = _name;
var fFileNeeded = false;
if (!CacheDirectory.FileExists(fileName))
{
fFileNeeded = true;
}
else
{
long cachedLength = CacheDirectory.FileLength(fileName);
long primaryLength = PrimaryDirectory.FileLength(fileName);
if (cachedLength != primaryLength)
{
fFileNeeded = true;
}
else
{
// cachedLastModifiedUTC was not ouputting with a date (just time) and the time was always off
var cachedFilePath = Path.Combine(_syncDirectory.CacheDirectoryPath, fileName);
var primaryFilePath = Path.Combine(_syncDirectory.PrimaryDirectoryPath, fileName);
var cachedLastModified = File.GetLastWriteTimeUtc(cachedFilePath);
var primaryLastModified = File.GetLastWriteTimeUtc(primaryFilePath);
if (cachedLastModified != primaryLastModified)
{
var timeSpan = primaryLastModified.Subtract(cachedLastModified);
if (timeSpan.TotalSeconds > 1)
{
fFileNeeded = true;
}
else
{
#if FULLDEBUG
Debug.WriteLine(timeSpan.TotalSeconds);
#endif
// file not needed
}
}
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if (fFileNeeded)
{
var primaryInput = PrimaryDirectory.OpenInput(fileName, IOContext.DEFAULT);
byte[] primaryInputBytes = new byte[primaryInput.Length];
primaryInput.ReadBytes(primaryInputBytes, 0, (int)primaryInputBytes.Length);
using (var cachedOutput = CacheDirectory.CreateOutput(fileName, IOContext.DEFAULT))
{
cachedOutput.WriteBytes(primaryInputBytes, (int)primaryInputBytes.Length);
cachedOutput.Flush();
}
primaryInput.Dispose();
var cachedFilePath = Path.Combine(_syncDirectory.CacheDirectoryPath, fileName);
var primaryFilePath = Path.Combine(_syncDirectory.PrimaryDirectoryPath, fileName);
var primaryLastModified = File.GetLastWriteTimeUtc(primaryFilePath);
File.SetLastWriteTimeUtc(cachedFilePath, primaryLastModified);
//using (var fileStream = _azureDirectory.CreateCachedOutputAsStream(fileName))
//{
// // get the blob
// _blob.DownloadToStream(fileStream);
// fileStream.Flush();
// Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, fileStream.Length));
//}
// and open it as an input
_indexInput = CacheDirectory.OpenInput(fileName, IOContext.DEFAULT);
}
else
{
#if FULLDEBUG
Debug.WriteLine(String.Format("Using cached file for {0}", _name));
#endif
// open the file in read only mode
_indexInput = CacheDirectory.OpenInput(fileName, IOContext.DEFAULT);
}
}
finally
{
_fileMutex.ReleaseMutex();
}
}
public SyncIndexInput(SyncDirectory directory, string name)
{
if (directory == null)
{
throw new ArgumentNullException(nameof(directory));
}
_name = name;
_syncDirectory = directory;
#if FULLDEBUG
Trace.WriteLine(String.Format("opening {0} ", _name));
#endif
_fileMutex = SyncMutexManager.GrabMutex(_syncDirectory, _name);
_fileMutex.WaitOne();
try
{
var fileName = _name;
var fFileNeeded = false;
if (!CacheDirectory.FileExists(fileName))
{
fFileNeeded = true;
}
else
{
long cachedLength = CacheDirectory.FileLength(fileName);
long masterLength = MasterDirectory.FileLength(fileName);
if (cachedLength != masterLength)
{
fFileNeeded = true;
}
else
{
long cacheDate = CacheDirectory.FileModified(fileName);
long masterDate = MasterDirectory.FileModified(fileName);
//we need to compare to the second instead of by ticks because when we call
//TouchFile in SyncIndexOutput this won't set the files to be identical
DateTime start = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc);
var cachedLastModifiedUTC = start.AddMilliseconds(cacheDate).ToUniversalTime();
var masterLastModifiedUTC = start.AddMilliseconds(masterDate).ToUniversalTime();
if (cachedLastModifiedUTC != masterLastModifiedUTC)
{
var timeSpan = masterLastModifiedUTC.Subtract(cachedLastModifiedUTC);
//NOTE: This heavily depends on TouchFile in SyncIndexOutput which sets both the
//master and slave files to be 'Now', in theory this operation shouldn't
//make the file times any bigger than 1 second
if (timeSpan.TotalSeconds > 2)
{
fFileNeeded = true;
}
else
{
#if FULLDEBUG
Debug.WriteLine(timeSpan.TotalSeconds);
#endif
// file not needed
}
}
}
}
// if the file does not exist
// or if it exists and it is older then the lastmodified time in the blobproperties (which always comes from the blob storage)
if (fFileNeeded)
{
//get the master file stream
using (var masterStream = new StreamInput(MasterDirectory.OpenInput(fileName)))
using (var cacheStream = _syncDirectory.CreateCachedOutputAsStream(fileName))
{
//copy this to the cached file stream
masterStream.CopyTo(cacheStream);
cacheStream.Flush();
Debug.WriteLine(string.Format("GET {0} RETREIVED {1} bytes", _name, cacheStream.Length));
}
// and open it as an input
_indexInput = CacheDirectory.OpenInput(fileName);
}
else
{
#if FULLDEBUG
Debug.WriteLine(String.Format("Using cached file for {0}", _name));
#endif
// open the file in read only mode
_indexInput = CacheDirectory.OpenInput(fileName);
}
}
finally
{
_fileMutex.ReleaseMutex();
}
}
