public DataReader(
revCore.Config revConfig,
string imageList
)
{
_imageList = imageList;
var statusDir = Path.GetDirectoryName(imageList);
var justFileName = Path.GetFileNameWithoutExtension(imageList);
var statusFile = Path.Combine(statusDir, $"{justFileName}.doneStatus.txt");
Console.WriteLine($"using status file {statusFile}");
if (File.Exists(statusFile))
{
using (var file = new System.IO.StreamReader(statusFile))
{
string line;
while ((line = file.ReadLine()) != null)
{
_doneMap[line] = true;
}
}
}
_statusWriter = File.AppendText(statusFile);
_rev = new revCore.Rev(revConfig);
Console.WriteLine("All initialized");
}
public async Task ImportDataAsync()
{
Console.WriteLine("Starting import");
var rev = new revCore.Rev(_appconfig);
var indexRegex = string.IsNullOrWhiteSpace(_importConfig.indexRegex) ? null :
new Regex(_importConfig.indexRegex);
var skipcount = 0;
using (StreamWriter sw = new StreamWriter(_doneFilename, true))
{
foreach (var fi in _fileGetter.fileToImport_imageRoot())
{
if (_doneFilename == fi.FullName)
{
continue;
}
if (!string.IsNullOrWhiteSpace(_importConfig.logFileLocation) && fi.FullName.StartsWith(_importConfig.logFileLocation))
{
//we donot want to import log files
continue;
}
if (_doneFileMap.ContainsKey(fi.FullName))
{
if ((skipcount++) % 10 == 0)
{
Console.WriteLine($"Skip count -> {skipcount}");
}
_donecount++;
continue;
}
try
{
var fields = new Dictionary <string, string> {
{ "filename", fi.Name }
};
var repoName = _importConfig.repoName;
if (null != indexRegex)
{
var match = indexRegex.Match(fi.FullName);
if (!match.Success)
{
_logger.LogError("regex did not match");
throw new Exception("regex did not match");
}
fields = match.Groups.Values.ToDictionary(k => k.Name, v => v.Value);
//Remove the full match group
fields.Remove(key: "0");
if (fields.ContainsKey(MyConfig.REPONAME_KEYWORD))
{
repoName = fields[MyConfig.REPONAME_KEYWORD];
fields.Remove(MyConfig.REPONAME_KEYWORD);
}
if (null != _importConfig.indexOverride)
{
foreach (var kv in _importConfig.indexOverride)
{
if (fields.ContainsKey(kv.key))
{
fields[kv.value] = fields[kv.key];
fields.Remove(kv.key);
}
}
}
}
await rev.CreateDocument(_importConfig.repoName,
fields,
new[] { fi }
);
//_logger.LogInformation($"Created documnent in {_importConfig.repoName} with page {fi.FullName}");
}
catch (Exception ex)
{
Console.Error.WriteLine($"failed to import file {fi.FullName}");
Console.Error.Write(ex.ToString());
_logger.LogError(ex.ToString());
continue;
}
sw.WriteLine(fi.FullName);
sw.Flush();
if (0 == (_donecount++) % 10)
{
Console.WriteLine($"done count -> {_donecount}");
}
if (_importConfig.removeAfterImport)
{
_fileGetter.RemoveFile(fi.FullName);
}
}
}
}
//code review Deepay attention to this error. If nothing is async just use TASK and return Task.completed.
public async Task ExportDocAsync()
{
var rev = new revCore.Rev(_rev);
// get list of documents
var res = rev.SearchDocs(_myconfig.projectName, new Dictionary <string, string> {
{ _myconfig.searchIndex, _myconfig.searchIndexValue }
}).ToArray();
// Regular expression to remove special charater document name
string pattern = @"([\\//;:?*><|])";
foreach (var doc in res)
{
string docname = "";
foreach (var i in doc.indexes)
{
if (i.Key == _myconfig.docFolderName1 || i.Key == _myconfig.docFolderName2)
{
docname = docname + i.Value + " ";
}
}
string DocName = Regex.Replace(docname, pattern, "") + doc.pages.Count();
createFolder($"{_myconfig.downFolder}\\{_myconfig.projectName}\\{DocName}");
//dee code review this is inefficient
/*
* foreach (var page in doc.pages)
* {
* var pathS = $"{_rev.revUrl}{page.path}";
* var docName = Regex.Replace(page.id, pattern, "");
* var pathD = $"C:\\{_myconfig.downFolder}\\{_myconfig.projectName}\\{DocName}\\{docName} ";
* await downloadFileAsync(pathS, pathD);
* }
*/
/* this is compact code
* var done = await Task.WhenAll(doc.pages.Select(async page =>
* {
* var pathS = $"{_rev.revUrl}{page.path}";
* var docName = Regex.Replace(page.id, pattern, "");
* var pathD = $"C:\\{_myconfig.downFolder}\\{_myconfig.projectName}\\{DocName}\\{docName} ";
* await downloadFileAsync(pathS, pathD);
* return true;
* }));
*/
var downloadTasks = doc.pages.Select(async page =>
{
var pathS = $"{_rev.revUrl}{page.path}";
var docName = Regex.Replace(page.id, pattern, "");
var pathD = $"C:\\{_myconfig.downFolder}\\{_myconfig.projectName}\\{DocName}\\{docName} ";
await downloadFileAsync(pathS, pathD);
return(true);
});
var done = await Task.WhenAll(downloadTasks);
}
}
