public DataReader( revCore.Config revConfig, string imageList ) { _imageList = imageList; var statusDir = Path.GetDirectoryName(imageList); var justFileName = Path.GetFileNameWithoutExtension(imageList); var statusFile = Path.Combine(statusDir, $"{justFileName}.doneStatus.txt"); Console.WriteLine($"using status file {statusFile}"); if (File.Exists(statusFile)) { using (var file = new System.IO.StreamReader(statusFile)) { string line; while ((line = file.ReadLine()) != null) { _doneMap[line] = true; } } } _statusWriter = File.AppendText(statusFile); _rev = new revCore.Rev(revConfig); Console.WriteLine("All initialized"); }
public async Task ImportDataAsync() { Console.WriteLine("Starting import"); var rev = new revCore.Rev(_appconfig); var indexRegex = string.IsNullOrWhiteSpace(_importConfig.indexRegex) ? null : new Regex(_importConfig.indexRegex); var skipcount = 0; using (StreamWriter sw = new StreamWriter(_doneFilename, true)) { foreach (var fi in _fileGetter.fileToImport_imageRoot()) { if (_doneFilename == fi.FullName) { continue; } if (!string.IsNullOrWhiteSpace(_importConfig.logFileLocation) && fi.FullName.StartsWith(_importConfig.logFileLocation)) { //we donot want to import log files continue; } if (_doneFileMap.ContainsKey(fi.FullName)) { if ((skipcount++) % 10 == 0) { Console.WriteLine($"Skip count -> {skipcount}"); } _donecount++; continue; } try { var fields = new Dictionary <string, string> { { "filename", fi.Name } }; var repoName = _importConfig.repoName; if (null != indexRegex) { var match = indexRegex.Match(fi.FullName); if (!match.Success) { _logger.LogError("regex did not match"); throw new Exception("regex did not match"); } fields = match.Groups.Values.ToDictionary(k => k.Name, v => v.Value); //Remove the full match group fields.Remove(key: "0"); if (fields.ContainsKey(MyConfig.REPONAME_KEYWORD)) { repoName = fields[MyConfig.REPONAME_KEYWORD]; fields.Remove(MyConfig.REPONAME_KEYWORD); } if (null != _importConfig.indexOverride) { foreach (var kv in _importConfig.indexOverride) { if (fields.ContainsKey(kv.key)) { fields[kv.value] = fields[kv.key]; fields.Remove(kv.key); } } } } await rev.CreateDocument(_importConfig.repoName, fields, new[] { fi } ); //_logger.LogInformation($"Created documnent in {_importConfig.repoName} with page {fi.FullName}"); } catch (Exception ex) { Console.Error.WriteLine($"failed to import file {fi.FullName}"); Console.Error.Write(ex.ToString()); _logger.LogError(ex.ToString()); continue; } sw.WriteLine(fi.FullName); sw.Flush(); if (0 == (_donecount++) % 10) { Console.WriteLine($"done count -> {_donecount}"); } if (_importConfig.removeAfterImport) { _fileGetter.RemoveFile(fi.FullName); } } } }
//code review Deepay attention to this error. If nothing is async just use TASK and return Task.completed. public async Task ExportDocAsync() { var rev = new revCore.Rev(_rev); // get list of documents var res = rev.SearchDocs(_myconfig.projectName, new Dictionary <string, string> { { _myconfig.searchIndex, _myconfig.searchIndexValue } }).ToArray(); // Regular expression to remove special charater document name string pattern = @"([\\//;:?*><|])"; foreach (var doc in res) { string docname = ""; foreach (var i in doc.indexes) { if (i.Key == _myconfig.docFolderName1 || i.Key == _myconfig.docFolderName2) { docname = docname + i.Value + " "; } } string DocName = Regex.Replace(docname, pattern, "") + doc.pages.Count(); createFolder($"{_myconfig.downFolder}\\{_myconfig.projectName}\\{DocName}"); //dee code review this is inefficient /* * foreach (var page in doc.pages) * { * var pathS = $"{_rev.revUrl}{page.path}"; * var docName = Regex.Replace(page.id, pattern, ""); * var pathD = $"C:\\{_myconfig.downFolder}\\{_myconfig.projectName}\\{DocName}\\{docName} "; * await downloadFileAsync(pathS, pathD); * } */ /* this is compact code * var done = await Task.WhenAll(doc.pages.Select(async page => * { * var pathS = $"{_rev.revUrl}{page.path}"; * var docName = Regex.Replace(page.id, pattern, ""); * var pathD = $"C:\\{_myconfig.downFolder}\\{_myconfig.projectName}\\{DocName}\\{docName} "; * await downloadFileAsync(pathS, pathD); * return true; * })); */ var downloadTasks = doc.pages.Select(async page => { var pathS = $"{_rev.revUrl}{page.path}"; var docName = Regex.Replace(page.id, pattern, ""); var pathD = $"C:\\{_myconfig.downFolder}\\{_myconfig.projectName}\\{DocName}\\{docName} "; await downloadFileAsync(pathS, pathD); return(true); }); var done = await Task.WhenAll(downloadTasks); } }