private List <TagFile> GetFiles(Organization org, string syncFolder)
{
Log.DebugFormat("Found synch folder for {0}", syncFolder);
DirParams dirParams = new DirParams
{
filespaceid = org.filespaceId,
path = syncFolder,
recursive = true,
filterfolders = true,
filemasklist = "*.tag"
};
var result = ExecuteRequest(Ticket, Method.GET, "/tcc/Dir", dirParams, typeof(DirResult));
DirResult dirResult = result as DirResult;
if (dirResult != null)
{
if (dirResult.success)
{
List <TagFile> files = new List <TagFile>();
GetTagFiles(dirResult, files, syncFolder);
return(files);
}
CheckForInvalidTicket(dirResult, "GetFiles");
}
else
{
Log.ErrorFormat("Null result from GetFiles for {0}", syncFolder);
}
return(null);
}
private void GetTagFiles(DirResult entry, List <TagFile> files, string path)
{
if (entry.isFolder)
{
if (!entry.leaf && entry.entries != null)
{
foreach (var e in entry.entries)
{
GetTagFiles(e, files, path);
}
}
}
else
{
if (OrgsHandler.FilenameDumpEnabled)
{
Log.LogDebug("Dumping {0} : {1}", entry.entryName, entry.createTime);
}
if (entry.leaf)
{
files.Add(new TagFile
{
fullName = $"{path}/{entry.entryName}",
createdUTC = entry.createTime
});
}
}
}
public void DirResult(DirResult dirResult)
{
// say we did a thing
Q.Add(new SnafflerMessage
{
DateTime = DateTime.Now,
DirResult = dirResult,
Type = SnafflerMessageType.DirResult
});
}
/// <summary>
/// Constructor<br>
/// [Description]<br>
/// none<br>
/// [Notes]<br>
/// </summary>
public FaceResult(HVC_RES parent)
: base(parent)
{
this.Parent = parent;
Dir = new DirResult(this);
Age = new AgeResult(this);
Gen = new GenResult(this);
Gaze = new GazeResult(this);
Blink = new BlinkResult(this);
Exp = new ExpResult(this);
}
/// <summary>
/// Get the FileCreated and Updated UTCs
/// and checks that the file exists.
/// </summary>
/// <returns></returns>
public static async Task <DirResult> GetFileInfoFromTccRepository(BusinessCenterFile sourceFile,
ILogger log, IServiceExceptionHandler serviceExceptionHandler, IFileRepository fileRepo)
{
DirResult fileEntry = null;
try
{
log.LogInformation(
$"GetFileInfoFromTccRepository: GetFileList filespaceID: {sourceFile.FileSpaceId} tccPathSource: {sourceFile.Path} sourceFile.Name: {sourceFile.Name}");
var dirResult = await fileRepo.GetFileList(sourceFile.FileSpaceId, sourceFile.Path, sourceFile.Name);
log.LogInformation(
$"GetFileInfoFromTccRepository: GetFileList dirResult: {JsonConvert.SerializeObject(dirResult)}");
if (dirResult == null || dirResult.entries.Length == 0)
{
serviceExceptionHandler.ThrowServiceException(HttpStatusCode.InternalServerError, 94, "fileRepo.GetFileList");
}
else
{
fileEntry = dirResult.entries.FirstOrDefault(f =>
!f.isFolder && (string.Compare(f.entryName, sourceFile.Name, true, CultureInfo.InvariantCulture) == 0));
if (fileEntry == null)
{
serviceExceptionHandler.ThrowServiceException(HttpStatusCode.InternalServerError, 94,
"fileRepo.GetFileList");
}
}
}
catch (Exception e)
{
serviceExceptionHandler.ThrowServiceException(HttpStatusCode.InternalServerError, 94, "fileRepo.GetFileList",
e.Message);
}
return(fileEntry);
}
private void GetTagFiles(DirResult entry, List <TagFile> files, string path)
{
if (entry.isFolder)
{
if (!entry.leaf && entry.entries != null)
{
foreach (var e in entry.entries)
{
GetTagFiles(e, files, path);
}
}
}
else
{
if (entry.leaf)
{
files.Add(new TagFile
{
fullName = $"{path}/{entry.entryName}",
createdUTC = entry.createTime
});
}
}
}
public void WalkTree(string currentDir)
{
// Walks a tree checking files and generating results as it goes.
if (!Directory.Exists(currentDir))
{
return;
}
try
{
string[] files = Directory.GetFiles(currentDir);
// check if we actually like the files
foreach (string file in files)
{
FileTaskScheduler.New(() =>
{
try
{
FileScanner.ScanFile(file);
}
catch (Exception e)
{
Mq.Error("Exception in FileScanner task for file " + file);
Mq.Trace(e.ToString());
}
});
}
}
catch (UnauthorizedAccessException)
{
//Mq.Trace(e.ToString());
//continue;
}
catch (DirectoryNotFoundException)
{
//Mq.Trace(e.ToString());
//continue;
}
catch (IOException)
{
//Mq.Trace(e.ToString());
//continue;
}
catch (Exception e)
{
Mq.Degub(e.ToString());
//continue;
}
try
{
string[] subDirs = Directory.GetDirectories(currentDir);
// Create a new treewalker task for each subdir.
if (subDirs.Length >= 1)
{
foreach (string dirStr in subDirs)
{
bool scanDir = true;
foreach (ClassifierRule classifier in MyOptions.DirClassifiers)
{
try
{
DirClassifier dirClassifier = new DirClassifier(classifier);
DirResult dirResult = dirClassifier.ClassifyDir(dirStr);
if (dirResult.ScanDir == false)
{
scanDir = false;
break;
}
}
catch (Exception e)
{
Mq.Trace(e.ToString());
continue;
}
}
if (scanDir == true)
{
TreeTaskScheduler.New(() =>
{
try
{
WalkTree(dirStr);
}
catch (Exception e)
{
Mq.Error("Exception in TreeWalker task for dir " + dirStr);
Mq.Error(e.ToString());
}
});
}
else
{
Mq.Trace("Skipped scanning on " + dirStr + " due to Discard rule match.");
}
}
}
}
catch (UnauthorizedAccessException)
{
//Mq.Trace(e.ToString());
//continue;
}
catch (DirectoryNotFoundException)
{
//Mq.Trace(e.ToString());
//continue;
}
catch (IOException)
{
//Mq.Trace(e.ToString());
//continue;
}
catch (Exception e)
{
Mq.Trace(e.ToString());
//continue;
}
}
public void WalkTree(string shareRoot)
{
try
{
// Walks a tree checking files and generating results as it goes.
Stack <string> dirs = new Stack <string>(20);
if (!Directory.Exists(shareRoot))
{
return;
}
dirs.Push(shareRoot);
while (dirs.Count > 0)
{
string currentDir = dirs.Pop();
string[] subDirs;
try
{
subDirs = Directory.GetDirectories(currentDir);
}
catch (UnauthorizedAccessException)
{
//Mq.Trace(e.ToString());
continue;
}
catch (DirectoryNotFoundException)
{
//Mq.Trace(e.ToString());
continue;
}
catch (Exception e)
{
Mq.Trace(e.ToString());
continue;
}
string[] files = null;
try
{
files = Directory.GetFiles(currentDir);
}
catch (UnauthorizedAccessException)
{
//Mq.Trace(e.ToString());
continue;
}
catch (DirectoryNotFoundException)
{
//Mq.Trace(e.ToString());
continue;
}
catch (Exception e)
{
Mq.Trace(e.ToString());
continue;
}
// check if we actually like the files
foreach (string file in files)
{
FileTaskScheduler.New(() =>
{
try
{
FileScanner fileScanner = new FileScanner(file);
}
catch (Exception e)
{
Mq.Trace(e.ToString());
}
});
}
// Push the subdirectories onto the stack for traversal if they aren't on any discard-lists etc.
foreach (string dirStr in subDirs)
{
foreach (ClassifierRule classifier in MyOptions.DirClassifiers)
{
try
{
DirClassifier dirClassifier = new DirClassifier(classifier);
DirResult dirResult = dirClassifier.ClassifyDir(dirStr);
// TODO: concurrency uplift: when there is a pooled concurrency queue, just add the dir as a job to the queue
if (dirResult.ScanDir)
{
dirs.Push(dirStr);
}
}
catch (Exception e)
{
Mq.Trace(e.ToString());
continue;
}
}
}
}
}
catch (Exception e)
{
Mq.Error(e.ToString());
}
}
private List <string> GetFolders(Organization org, string path)
{
Log.DebugFormat("GetFolders: org={0} {1}, path={2}", org.shortName, org.filespaceId, path);
List <string> folders = null;
try
{
//Get list of folders one level down from path
DirParams dirParams = new DirParams
{
filespaceid = org.filespaceId,
path = path,
recursive = false,
filterfolders = true,
filemasklist = "*.tag"
};
var result = ExecuteRequest(Ticket, Method.GET, "/tcc/Dir", dirParams, typeof(DirResult));
DirResult dirResult = result as DirResult;
if (dirResult != null)
{
if (dirResult.success)
{
if (dirResult.entries != null)
{
var folderEntries = (from d in dirResult.entries
where d.isFolder && !d.leaf
select d).ToList();
folders = new List <string>();
foreach (var folderEntry in folderEntries)
{
string lPath;
if (path == "/")
{
lPath = string.Format("{0}{1}", path, folderEntry.entryName);
}
else
{
lPath = string.Format("{0}/{1}", path, folderEntry.entryName);
}
folders.Add($"/{folderEntry.entryName}");
}
if (!folders.Any())
{
Log.WarnFormat("No folders found for org {0}", org.shortName);
}
}
else
{
Log.WarnFormat("No folders returned from GetFolders for org {0}", org.shortName);
folders = new List <string>();
}
}
else
{
CheckForInvalidTicket(dirResult, "GetFolders");
}
}
else
{
Log.ErrorFormat("Null result from GetFolders for org {0}", org.shortName);
}
}
catch (Exception ex)
{
Log.ErrorFormat("Failed to get list of TCC folders: {0}", ex.Message);
}
return(folders);
}
public void WalkTree(string currentDir, int delayTime = 1)
{
// Walks a tree checking files and generating results as it goes.
if (!Directory.Exists(currentDir))
{
return;
}
try
{
string[] files = Directory.GetFiles(currentDir);
// check if we actually like the files
foreach (string file in files)
{
//FileTaskScheduler.New(() =>
//{
try
{
FileScanner.ScanFile(file);
}
catch (Exception e)
{
Mq.Error("Exception in FileScanner task for file " + file);
Mq.Trace(e.ToString());
}
//Mq.Info("Sleeping " + delayTime.ToString() + " Seconds");
System.Threading.Thread.Sleep(delayTime);
//});
}
}
catch (UnauthorizedAccessException)
{
//Mq.Trace(e.ToString());
//continue;
}
catch (DirectoryNotFoundException)
{
//Mq.Trace(e.ToString());
//continue;
}
catch (Exception e)
{
Mq.Trace(e.ToString());
//continue;
}
try
{
string[] subDirs = Directory.GetDirectories(currentDir);
// Create a new treewalker task for each subdir.
if (subDirs.Length >= 1)
{
foreach (string dirStr in subDirs)
{
Mq.Degub($"Processing directory {dirStr}");
foreach (ClassifierRule classifier in MyOptions.DirClassifiers)
{
try
{
DirClassifier dirClassifier = new DirClassifier(classifier);
DirResult dirResult = dirClassifier.ClassifyDir(dirStr);
if (dirResult.ScanDir)
{
//TreeTaskScheduler.New(() =>
//{
try
{
WalkTree(dirStr, delayTime);
}
catch (Exception e)
{
Mq.Error("Exception in TreeWalker task for dir " + dirStr);
Mq.Error(e.ToString());
}
//});
}
}
catch (Exception e)
{
Mq.Trace(e.ToString());
continue;
}
// Mq.Info("Sleeping " + delayTime.ToString() + " Seconds");
System.Threading.Thread.Sleep(delayTime);
}
}
}
}
catch (UnauthorizedAccessException)
{
//Mq.Trace(e.ToString());
//continue;
}
catch (DirectoryNotFoundException)
{
//Mq.Trace(e.ToString());
//continue;
}
catch (Exception e)
{
Mq.Trace(e.ToString());
//continue;
}
}
