async Task pushFileToMeta(List <MetaSegment.Command.FileOrigin> metaSegments, long fileSize, string remotePath, bool ignoreFile = false)
{
await metaSem.WaitAsync();
try
{
// validate and resolve remotePath
// MAYBE: move remotePath conversion out of here and replace here with validation after conversion
if (Path.DirectorySeparatorChar != '/' && Path.AltDirectorySeparatorChar != '/')
{
throw new NotImplementedException("remotePath separator must be '/'");
}
if (remotePath.Substring(0, 1) != "/")
{
throw new ArgumentException("Invalid path. Must be absolute file path.");
}
remotePath = Path.GetFullPath(remotePath);
if (remotePath.Substring(0, 1) != "/" || !Path.IsPathRooted(remotePath) || Path.GetFileName(remotePath) == "")
{
throw new ArgumentException("Invalid path. Must be absolute file path.");
}
// convert to actual remote path
// split path into partial paths: root (empty), directories (without leading or trailing slash) and file name
var root = "";
var candidateDirs = ((Func <string, string[]>)(path =>
{
var pp = Path.GetDirectoryName(path).Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries);
if (pp.Length == 0)
{
return(pp);
}
pp[0] = $"{pp[0]}";
for (var i = 1; i < pp.Length; i++)
{
pp[i] = $"{pp[i - 1]}/{pp[i]}";
}
return(pp);
}))(remotePath);
var file = remotePath.Substring(1);
var allDirs = new string[] { root }.Concat(candidateDirs).ToArray();
var all = allDirs.Concat(new[] { file }).ToArray();
// check if we can push to meta at that remotePath.
// if we want deletion or modification later, we need to iterate over the index here and aggregate meta
foreach (var dir in candidateDirs)
{
if (DB.SQLMap.CommandMetaType.File == db.MetaTypeAtPathInTransientCache(dir) ||
null != db.FindMatchingSegmentInAssurancesByIndexId(generator.GenerateMetaFileID(0, dir)))
{
throw new MetaEntryOverwriteException($"Directory '{dir}' would overwrite file at the same path.");
}
}
var metaTypeFile = db.MetaTypeAtPathInTransientCache(file);
if (DB.SQLMap.CommandMetaType.Folder == metaTypeFile ||
null != db.FindMatchingSegmentInAssurancesByIndexId(generator.GenerateMetaFolderID(0, file)))
{
throw new MetaEntryOverwriteException($"File '{file}' would overwrite folder at the same path.");
}
if (DB.SQLMap.CommandMetaType.File == metaTypeFile ||
null != db.FindMatchingSegmentInAssurancesByIndexId(generator.GenerateMetaFileID(0, file)))
{
throw new MetaEntryOverwriteException($"File '{file}' would overwrite file at the same path.");
}
Constants.Logger.Log($"Pushing to meta: {remotePath}");
var allDirsEx = allDirs.Select((d, i) => new { last = Path.GetFileName(d), full = d, i }).ToArray();
var pushList = new List <DB.SQLMap.Command>();
foreach (var dir in allDirsEx)
{
var commands = new List <DB.SQLMap.Command>();
var i = 0;
for (; ; i++)
{
var indexId = generator.GenerateMetaFolderID((uint)i, dir.full);
var seg = db.FindMatchingSegmentInAssurancesByIndexId(indexId);
if (seg == null)
{
break;
}
var chunk = await DownloadChunk(indexId); // TODO cache
commands.AddRange(MetaSegment.FromByteArray(chunk).Commands
.Select(c => c.ToDBObject())
.Select(c =>
{
c.IsNew = false;
c.Path = dir.full;
c.MetaType = DB.SQLMap.CommandMetaType.Folder;
return(c);
})
);
}
var commands2 = db.CommandsInTransientCache(dir.full);
if (commands2.Where(c => c.Index < i).FirstOrDefault() != null)
{
throw new InvalidDataException($"too small index in meta db cache for dir '{dir}'");
}
i += commands2.Count;
var allCommands = commands.Concat(commands2).ToArray();
if (dir.i != allDirsEx.Length - 1)
{
var next = allDirsEx[dir.i + 1];
var hasFolder = null != allCommands.Where(c => c.MetaType == DB.SQLMap.CommandMetaType.Folder && c.FolderOrigin_Name == next.last).FirstOrDefault();
if (hasFolder)
{
continue;
}
// add folder to folder
pushList.Add(new DB.SQLMap.Command
{
IsNew = true,
Path = dir.full,
Index = i,
MetaType = DB.SQLMap.CommandMetaType.Folder,
CMD = DB.SQLMap.Command.CMDV.ADD,
TYPE = DB.SQLMap.Command.TYPEV.FOLDER,
FolderOrigin_Name = next.last,
});
}
else
{
if (!ignoreFile)
{
var fileName = Path.GetFileName(file);
var hasFile = null != allCommands.Where(c => c.MetaType == DB.SQLMap.CommandMetaType.Folder && c.FolderOrigin_Name == fileName).FirstOrDefault();
if (hasFile)
{
throw new MetaEntryOverwriteException($"File '{file}' would overwrite file in parent folder.");
}
// add file to folder
pushList.Add(new DB.SQLMap.Command
{
IsNew = true,
Path = dir.full,
Index = i,
MetaType = DB.SQLMap.CommandMetaType.Folder,
CMD = DB.SQLMap.Command.CMDV.ADD,
TYPE = DB.SQLMap.Command.TYPEV.FILE,
FolderOrigin_Name = fileName,
FolderOrigin_FileSize = fileSize,
});
}
}
}
if (!ignoreFile)
{
// add blocks to file
pushList.AddRange(metaSegments.Select((ms, i) => new DB.SQLMap.Command
{
IsNew = true,
Path = file,
Index = i,
MetaType = DB.SQLMap.CommandMetaType.File,
CMD = DB.SQLMap.Command.CMDV.ADD,
TYPE = DB.SQLMap.Command.TYPEV.BLOCK,
FileOrigin_Hash = ms.Hash,
FileOrigin_Size = ms.Size,
FileOrigin_Start = ms.Start,
}));
}
db.AddCommandsToTransientCache(pushList);
}
finally
{
metaSem.Release();
}
}
public async Task <Meta> DownloadMetaForPath(string path)
{
int maxConcurrentTasks = 10;
uint metaIndex = 0;
var tasks = new List <Task <byte[]> >();
var cmdsInTransientCache = db.CommandsInTransientCache(path);
var first = cmdsInTransientCache.FirstOrDefault();
var isFile = first?.MetaType == DB.SQLMap.CommandMetaType.File ||
null != db.FindMatchingSegmentInAssurancesByIndexId(generator.GenerateMetaFileID(0, path));
if (!isFile)
{
var isFolder = first?.MetaType == DB.SQLMap.CommandMetaType.Folder ||
null != db.FindMatchingSegmentInAssurancesByIndexId(generator.GenerateMetaFolderID(0, path));
if (!isFolder)
{
return(null);
}
}
for (; ; metaIndex++)
{
var indexId = isFile
? generator.GenerateMetaFileID(metaIndex, path)
: generator.GenerateMetaFolderID(metaIndex, path);
var seg = db.FindMatchingSegmentInAssurancesByIndexId(indexId);
if (seg == null)
{
break;
}
var task = DownloadChunk(indexId);
tasks.Add(task);
if (tasks.Count == maxConcurrentTasks)
{
var t = await Task.WhenAny(tasks);
if (t.IsFaulted)
{
throw t.Exception; // catch earlier by capping maxConcurrentTasks to connection limit?
}
}
}
var combinedMeta = new Meta {
Path = path, IsFile = isFile
};
var values = await Task.WhenAll(tasks);
foreach (var cmds1 in values.Select(v => MetaSegment.FromByteArray(v).Commands))
{
combinedMeta.Commands.AddRange(cmds1);
}
var cmds2 = cmdsInTransientCache.OrderBy(c => c.Index).Select(x => x.ToProtoObject());
combinedMeta.Commands.AddRange(cmds2);
return(combinedMeta);
}