public void BackwardCountTest()
{
var iterator = Repository.ObjectFactory.GetDataFileIterator(FixtureRootRepoFolder, true);
iterator.Seek(DateTime.MaxValue);
Assert.IsNull(iterator.Previous, "Before moving forward there can be no previous");
int count = 0;
IRepositoryFile previous = null;
while (iterator.Current != null)
{
if (count > 0)
{
Assert.Less(iterator.Current.Name.LastItemTimestamp, previous.Name.FirstItemTimestamp);
}
++count;
previous = iterator.Current;
iterator.MoveNext();
Assert.AreSame(previous, iterator.Previous);
}
Assert.AreEqual(_expectedFileCount, count);
}
/// <summary>
/// Find first leaf data [sub]folder and data file in it containing items dated at, earlier or later than the
/// <paramref name="seekTime"/>
/// </summary>
/// <param name="seekTime">
/// Specifies the start of date-time range (inclusive) to find data in
/// </param>
/// <param name="backwards">
/// To which side from <paramref name="seekTime"/> to look for data
/// </param>
/// <returns>
/// First (according to <paramref name="seekTime"/>) data file containing the required data;
/// <see langword="null"/> if none found
/// </returns>
public IRepositoryFile Seek(DateTime seekTime, bool backwards)
{
_log.DebugFormat("Seeking data folder for reader, descr: {0}"
, this.IsVirtualRoot ? "Virtual Root" : this._folderDescriptor.ToString());
IRepositoryFileName dataFile = null;
IRepositoryFile retval = null;
if (IsLeafFolder)
{
LoadFiles(false);
dataFile = _dataFileBrowser.GetFile(seekTime, backwards);
if (null != dataFile)
{
_log.DebugFormat("Seek found file: {0}", dataFile.FileName);
retval = GetContainedRepositoryFile(dataFile);
}
}
else
{
LoadSubFolders(false);
IDirectedEnumerable <IDataFolder> sequence = GetSubfoldersSequence(seekTime, backwards);
for (
IEnumerator <IDataFolder> scanner = sequence.GetEnumerator();
scanner.MoveNext() && null == retval;
)
{
retval = scanner.Current.Seek(seekTime, backwards);
}
}
return(retval);
}
/// <summary>
/// Initialise iteration position.
/// </summary>
/// <param name="seekTime">
/// Data timestamp; files containig data items with the timestamp will be iterated.
/// </param>
/// <param name="backwards">
/// Direction in which to search for data files if the <paramref name="seekTime"/> is not covered by any
/// existing data file
/// </param>
/// <returns>
/// First found data file; null if none found.
/// </returns>
/// <remarks>
/// The data in a data file does not have to be entirely in the specified datetime range for the file to be iterated. If any data in the file
/// falls in the sought range it will be returned by the iterator.
/// </remarks>
/// <exception cref="ObjectDisposedException">
/// The target repository instance is disposed.
/// </exception>
/// <exception cref="InvalidOperationException">
/// The <paramref name="targetFolder"/> is detached.
/// </exception>
public IRepositoryFile Seek(DateTime seekTime, bool backwards)
{
RepositoryFolder.CheckNotDetached(Folder);
DateTime seekTimeCorrected;
IRepositoryFile currentFile = _targetFolder.RootDataFolder.Seek(seekTime, backwards);
if (currentFile != null)
{
seekTimeCorrected = currentFile.Name.FirstItemTimestamp;
}
else
{
seekTimeCorrected = seekTime;
}
SeekExact(seekTimeCorrected);
Check.Ensure(Current == null || Current.Name == currentFile.Name, "Seek results differ");
Check.Ensure(Current == null ||
GetComparer(backwards).Compare(
backwards ? Current.Name.FirstItemTimestamp : Current.Name.LastItemTimestamp
, seekTime
) >= 0
);
Check.Ensure(NextBackwards == null || NextBackwards.Name.End <= seekTime);
Check.Ensure(NextForward == null || NextForward.Name.FirstItemTimestamp > seekTime);
return(Current);
}
public UploadFileController(IRepositoryFile repository, IWebHostEnvironment environment, IMapper mapper, ILogger <UploadFileController> logger)
{
_repository = repository;
_environment = environment;
_mapper = mapper;
_logger = logger;
}
private IRepositoryFile GetNextNullSafe(IRepositoryFile file, bool backwards)
{
if (file == null)
{
return(null);
}
return(file.GetNext(backwards));
}
public async Task DeleteFile(IRepositoryFile file, string containerName)
{
CloudBlobContainer container = await this.GetContainer(containerName, this.ConnectionStringName).ConfigureAwait(false);
string location = file.FileLocation.Substring(file.FileLocation.IndexOf(containerName, StringComparison.Ordinal) + containerName.Length).Trim('/');
CloudBlockBlob blockBlob = container.GetBlockBlobReference(location);
await blockBlob.DeleteAsync().ConfigureAwait(false);
}
/// <summary>
/// Get timestamp of the first (chronologically) data item in the subtree with this data folder at its root.
/// </summary>
/// <param name="fromEnd">
/// The direction of search.
/// <see langword="true"/> - search for last (chronologically) data item
/// <see langword="false"/> - search for first (chronologically) data item
/// </param>
/// <returns>
/// <see cref="DateTime.MinValue"/> - there's no data and <paramref name="fromEnd"/> is <see langword="true"/>
/// <see cref="DateTime.MaxValue"/> - there's no data and <paramref name="fromEnd"/> is <see langword="false"/>
/// Otherwise timestamp of first existing data item.
/// </returns>
public DateTime GetFirstItemTimestamp(bool fromEnd)
{
IRepositoryFile firstFile = FindFirstDataFile(fromEnd);
if (null == firstFile)
{
return(fromEnd ? DateTime.MinValue : DateTime.MaxValue);
}
return(fromEnd ? firstFile.Name.LastItemTimestamp : firstFile.Name.FirstItemTimestamp);
}
private IRepositoryFile GetContainedRepositoryFile(IRepositoryFileName fileName)
{
IRepositoryFile retval = null;
if (null != fileName)
{
retval = new RepositoryFile(containingFolder: this, fileName: fileName);
}
return(retval);
}
public async Task <byte[]> GetFile(IRepositoryFile repositoryFile, string containerName)
{
byte[] result;
using (FileStream stream = File.Open(repositoryFile.FileLocation, FileMode.Open))
{
result = new byte[stream.Length];
await stream.ReadAsync(result, 0, (int)stream.Length).ConfigureAwait(false);
}
return(result);
}
public async Task <byte[]> GetFile(IRepositoryFile file, string containerName)
{
CloudBlobContainer container = await this.GetContainer(containerName, this.ConnectionStringName).ConfigureAwait(false);
CloudBlockBlob blockBlob = container.GetBlockBlobReference(file.FileName);
MemoryStream stream = new MemoryStream();
await blockBlob.DownloadToStreamAsync(stream).ConfigureAwait(false);
return(stream.ToArray());
}
/// <summary>
/// Find data file owning/covering the <paramref name="seekTime"/>
/// </summary>
/// <param name="seekTime">
/// Data timestamp
/// </param>
/// <returns>
/// Existing <see cref="IRepositoryFile"/> covering by its date-time range the <paramref name="seekTime"/>.
/// <see langword="null"/> if such file does not exist.
/// </returns>
public IRepositoryFile SeekOwner(DateTime seekTime)
{
IRepositoryFile file = Seek(seekTime, false);
if (file != null && file.Name.FirstItemTimestamp > seekTime)
{
// file is not covering the timestamp
file = null;
}
return(file);
}
public async Task SaveFile(IRepositoryFile repositoryFile, Stream fileStream, string containerName)
{
string fileLocation = Path.Combine(this.RootFolder, "FileRepo", containerName, Guid.NewGuid().ToString(), repositoryFile.FileName);
Directory.CreateDirectory(Path.GetDirectoryName(fileLocation));
using (Stream file = File.Create(fileLocation))
{
await fileStream.CopyToAsync(file).ConfigureAwait(false);
}
repositoryFile.FileLocation = fileLocation;
}
public async Task <byte[]> GetFile(IRepositoryFile file, string containerName)
{
CloudBlobContainer container = await this.GetContainer(containerName, this.ConnectionStringName).ConfigureAwait(false);
string location = file.FileLocation.Substring(file.FileLocation.IndexOf(containerName, StringComparison.Ordinal) + containerName.Length).Trim('/');
CloudBlockBlob blockBlob = container.GetBlockBlobReference(location);
MemoryStream stream = new MemoryStream();
await blockBlob.DownloadToStreamAsync(stream).ConfigureAwait(false);
return(stream.ToArray());
}
public async Task SaveFile(IRepositoryFile file, Stream fileStream, string containerName)
{
CloudBlobContainer container = await this.GetContainer(containerName, this.ConnectionStringName).ConfigureAwait(false);
CloudBlockBlob blockBlob = container.GetBlockBlobReference(file.FileName);
blockBlob.Properties.ContentType = this.ContentType;
fileStream.Position = 0;
await blockBlob.UploadFromStreamAsync(fileStream).ConfigureAwait(false);
file.FileLocation = blockBlob.Uri?.ToString();
}
/// <summary>
/// Cut data files in the subtree
/// </summary>
/// <param name="timestamp">
/// Cutting data timesatamp
/// </param>
/// <param name="predecessor">
/// Output, the immediate predecessor of the data file (if any) covering <paramref name="owner"/>.
/// </param>
/// <param name="owner">
/// Output, the data file covering the <paramref name="owner"/>.
/// </param>
/// <param name="successor">
/// Output, the immediate successor of the data file (if any) covering <paramref name="owner"/>.
/// </param>
public void CutDataFiles(DateTime timestamp, out IRepositoryFile predecessor, out IRepositoryFile owner, out IRepositoryFile successor)
{
IDataFolder ownerLeafFolder;
IDataFolder predecessorLeafFolder;
IDataFolder successorLeafFolder;
predecessor = owner = successor = null;
if (this.IsLeafFolder)
{
LoadFiles(false);
IRepositoryFileName fnPred, fnOwner, fnSucc;
this.DataFileBrowser.GetDataFiles(timestamp, out fnPred, out fnOwner, out fnSucc);
predecessor = GetContainedRepositoryFile(fnPred);
owner = GetContainedRepositoryFile(fnOwner);
successor = GetContainedRepositoryFile(fnSucc);
}
else
{
CutDescendantFolders(timestamp, Constants.DataFolderLevelLeaf, out predecessorLeafFolder, out ownerLeafFolder, out successorLeafFolder);
if (ownerLeafFolder != null)
{
ownerLeafFolder.CutDataFiles(timestamp, out predecessor, out owner, out successor);
}
if (predecessor == null && predecessorLeafFolder != null)
{
predecessor = predecessorLeafFolder.FindFirstDataFileInSegment(true);
}
if (successor == null && successorLeafFolder != null)
{
successor = successorLeafFolder.FindFirstDataFileInSegment(false);
}
}
}
/// <summary>
/// Move current file to the next file.
/// </summary>
/// <param name="backwards">
/// The direction in which to move
/// </param>
/// <returns>
/// New current file (either <see cref="NextForward"/> or <see cref="NextBackwards"/> prior to the call to this method)
/// </returns>
/// <remarks>
/// Does nothing if <see cref="Current"/> and <see cref="Next"/> are already <see langword="null"/>.
/// Otherwise the iterator will not be able to go back.
/// </remarks>
/// <exception cref="ObjectDisposedException">
/// The target repository instance is disposed.
/// </exception>
/// <exception cref="InvalidOperationException">
/// The <paramref name="targetFolder"/> is detached.
/// </exception>
public IRepositoryFile MoveNext(bool backwards)
{
RepositoryFolder.CheckNotDetached(Folder);
if (Current != null || Next != null)
{
Previous = Current;
if (backwards)
{
NextForward = Current;
Current = NextBackwards;
Next = NextBackwards = GetNextNullSafe(Current, backwards);
}
else
{
NextBackwards = Current;
Current = NextForward;
Next = NextForward = GetNextNullSafe(Current, backwards);
}
}
return(Current);
}
/// <summary>
/// Loads the specified file and positions at the first data item.
/// </summary>
/// <exception cref="System.IO.FileNotFoundException">
/// The <paramref name="dataFile"/> does not exist on disk.
/// </exception>
private void ReadFile(IRepositoryFile dataFile)
{
// passing null for equally timestamped items comparer, data items are read in the same
// order as they were written; no sorting when reading by design
IDataFileAccessor accessor = GetAccessor(dataFile);
// will throw exception if file not found or failed to read
accessor.ReadFromFile();
IList <IDataItem> dataList = accessor.GetAllItems();
if (Backwards)
{
_listReader = new BackwardListReader <IDataItem>(dataList);
}
else
{
_listReader = new ForwardListReader <IDataItem>(dataList);
}
this.LoadedFile = dataFile;
Check.Ensure(_listReader.HasItem);
Check.Ensure(CurrentItem != null);
}
/// <summary>
/// Get next data file in the same repo folder.
/// </summary>
/// <param name="backwards">
/// The direction in which to look for data file relative to this file: to the past or to the future
/// </param>
/// <returns>
/// Next data file or <see langword="null"/> if none exists.
/// </returns>
public IRepositoryFile GetNext(bool backwards)
{
IRepositoryFile retval = null;
IRepositoryFileName fileInSameFolder = ContainingFolder.GetNextDataFile(this.Name, backwards);
if (null != fileInSameFolder)
{
retval = new RepositoryFile(containingFolder: ContainingFolder, fileName: fileInSameFolder);
}
else
{
// scanning sibling leaf folders until first data file is found
for (
IDataFolder nextFolder = ContainingFolder.GetNextSiblingInTree(backwards);
nextFolder != null && retval == null;
nextFolder = nextFolder.GetNextSiblingInTree(backwards))
{
retval = nextFolder.FindFirstDataFile(backwards);
}
}
return(retval);
}
private IDataFileAccessor GetAccessor(IRepositoryFile targetFile)
{
return(Repository.ObjectFactory.GetDataFileAccessor(targetFile.ContainingFolder, targetFile.Name));
}
public void MyClassInitialize()
{
lock (GetType())
{
if (_emptyDataFolder == null)
{
const int itemsCount = 100000;
_daysPerFile = ((double)_dataItemsPerFile) * _itemsIntervalMinutes / 60.0 / 24.0;
// 200 * 20 minutes = 8000 minutes per file (5.55556 days)
DateTime firstTime = DateTime.Now.AddDays(-10);
_firstDataItemTime = firstTime;
//_expectedFileCount = (int)Math.Ceiling((double)itemsCount / (double)_dataItemsPerFile);
IFolder targetFolder = (IFolder)FixtureRootRepoFolder;
string targetFolderPath = targetFolder.FullPath;
targetFolder.Properties.DesiredItemsPerFile = _dataItemsPerFile;
using (IRepositoryWriter writer = targetFolder.GetWriter())
{
DateTime lastTime = DateTime.MinValue;
int n;
for (n = 0; n < itemsCount; ++n)
{
Mock.TestDataItem item = Mock.TestDataItem.GetTestItem(n);
lastTime = firstTime.AddMinutes(n * _itemsIntervalMinutes);
item.DateTime = lastTime;
writer.Write(item);
}
_lastDataItemTime = lastTime;
writer.Flush();
writer.Close();
}
for (
var dataFile = targetFolder.RootDataFolder.FindFirstDataFile(false);
dataFile != null;
dataFile = dataFile.GetNext(false), ++_expectedFileCount
)
{
}
Console.WriteLine("Expected file count enumerated via RepositoryFile: {0}", _expectedFileCount);
// data folder boundaries may split data files thus extra ones
Assert.GreaterOrEqual(_expectedFileCount, (int)Math.Ceiling((double)itemsCount / (double)_dataItemsPerFile), "Data file count unexpected");
// creating empty folder
IRepositoryFile file = targetFolder.RootDataFolder.Seek(firstTime.AddMinutes(itemsCount * _itemsIntervalMinutes / 3), false);
_emptyDataFolder = file.ContainingFolder;
for (
file = _emptyDataFolder.FindFirstDataFile(false);
file != null && file.ContainingFolder == _emptyDataFolder;
file = file.GetNext(false))
{
file.Delete();
--_expectedFileCount;
}
Assert.AreEqual(0, _emptyDataFolder.DataFileBrowser.FileCount);
Console.WriteLine("Expected file count after removing file by file: {0}", _expectedFileCount);
//
IDataFolder dfolderToDelete = _emptyDataFolder.ParentDataFolder.GetNextSiblingInTree(false).GetNextSiblingInTree(false);
Assert.AreEqual(1, dfolderToDelete.Level);
_deletedDataFolder = new RepoFileContainerDescriptor()
{
Start = dfolderToDelete.Start,
End = dfolderToDelete.End,
Level = dfolderToDelete.Level,
RelativePath = dfolderToDelete.RelativePath
};
_expectedFileCount -= dfolderToDelete.GetSubfolders(DateTime.MinValue, false).Sum((f) => f.DataFileBrowser.FileCount);
Console.WriteLine("Expected file count after removing data folder {0}: {1}", dfolderToDelete.PathInRepository, _expectedFileCount);
Console.WriteLine("Removing folder {0}", dfolderToDelete.PathInRepository);
dfolderToDelete.Delete(false);
Assert.IsFalse(dfolderToDelete.Exists);
}
} //lock
}
private void ConstructRepository(IRepositoryFile file)
{
RepositoryMode = file.RepositoryMode;
RepositorySha = file.RepositorySha;
}
public ServiceFile(IRepositoryFile repositoryFile)
{
_repositoryFile = repositoryFile;
}
public SeekFileResult(DateTime seekTime, IRepositoryFile file)
{
this.SeekDateTime = seekTime;
this.RepositoryFile = file;
}
