public async Task <bool> Export(AppRepository app, S3StorageRepository S3, Del message, Del threadMessage, Del uploadStatusMessage, Del uploadedMessage, Del skippedMessage, Del handleUploadCompleted)
{
IsRunning = true;
IsUploadingStop = false;
await LoadDocuments(app, message, threadMessage, uploadStatusMessage, uploadedMessage, skippedMessage, handleUploadCompleted);
return(false);
}
public async Task <bool> Export(AppRepository app, S3StorageRepository S3, Del message)
{
IsRunning = true;
LoadDocuments(app, message);
await ExportDocuments(app, S3, message);
IsRunning = false;
return(false);
}
private async Task CloseAndUpload(AppRepository app, S3StorageRepository S3, Del message, StreamWriter jsonwrite, string file_path, string file_name)
{
if (jsonwrite != null)
{
jsonwrite.Close();
StorageRequest storage = new StorageRequest
{
Name = file_name,
Type = "import",
EncryptionKeyId = null,
Path = file_path,
};
message("Uploading " + file_name);
await S3.UploadAsync(app, storage);
}
}
private async Task ExportDocuments(AppRepository app, S3StorageRepository S3, Del message)
{
var documents = Database.GetCollection <Document>("myob_documents");
var count = 0;
var file_index = 0;
var file_path = "";
var file_name = "";
StreamWriter jsonwrite = null;
var records = documents.FindAll();
foreach (var record in records)
{
if (count % 50000 == 0)
{
await CloseAndUpload(app, S3, message, jsonwrite, file_path, file_name);
file_name = "index_" + file_index + ".json";
message("Creating export file " + file_name);
file_path = Environment.CurrentDirectory + "\\" + file_name;
FileStream fs = File.Create(file_path);
fs.Close();
file_index++;
jsonwrite = new StreamWriter(file_path, true);
}
string json = JsonConvert.SerializeObject(record, Formatting.None);
jsonwrite.WriteLine(json);
count++;
}
await CloseAndUpload(app, S3, message, jsonwrite, file_path, file_name);
}
private async Task <bool> LoadDocuments(AppRepository app, Del message, Del threadMessage, Del uploadStatusMessage, Del uploadedMessage, Del skippedMessage, Del handleUploadCompleted)
{
app.LogMessage("FS: Getting documents remaining to be exported");
uploadStatusMessage("Getting documents remaining to be exported");
var fileList = new List <FileInfo>();
var folderList = new List <DirectoryInfo>();
foreach (string folder in app.Settings.FolderLists)
{
DirectoryInfo d = new DirectoryInfo(folder);
var r = d.GetFiles("*.*", SearchOption.AllDirectories);
fileList.AddRange(r);
uploadStatusMessage("Loading subfolders " + d.Name);
for (int i = 0; i < r.Count(); i++)
{
folderList.Add(d);
}
}
FileInfo[] Files = fileList.ToArray();
DirectoryInfo[] Dirs = folderList.ToArray();
var fileIndex = 0;
var completedThreads = 0;
var fileCounts = Files.Count();
var skipped = 0;
var uploaded = 0;
var errors = 0;
var documents = Database.GetCollection <Document>("documents");
documents.EnsureIndex(x => x.FilePath);
ThreadsNum = ThreadsNum < fileCounts ? ThreadsNum : fileCounts;
Thread[] threadsArray = new Thread[ThreadsNum];
for (int i = 0; i < ThreadsNum; i++)
{
threadsArray[i] = new Thread(async(object param) =>
{
int threadId = (int)param;
while (fileIndex < fileCounts)
{
if (IsUploadingStop)
{
Thread.Sleep(1000);
}
else
{
try
{
var activeIndex = fileIndex;
fileIndex++;
if (activeIndex < ThreadsNum)
{
activeIndex = threadId;
}
var file = Files[activeIndex];
Document document = new Document();
document.FilePath = file.FullName;
document.DateModified = file.LastWriteTimeUtc;
uploadStatusMessage(GetWorkingFolder(Dirs[activeIndex].ToString()));
var existing = documents.FindOne(x => x.FilePath == document.FilePath);
StorageRequest storage = new StorageRequest
{
Path = file.FullName,
Name = file.Name,
Type = makeCloudFolderPath(Dirs[activeIndex].ToString(), file.Name, file.FullName, app.Settings.IncludeFolderName),
EncryptionKeyId = null,
};
S3StorageRepository s3 = new S3StorageRepository(Database);
if (existing == null || existing.DateModified < document.DateModified)
{
var display = "";
var uploadResponse = false;
try
{
string statusofThread = MakeStatusOfThread(threadId, file.Length, file.Name);
threadMessage(statusofThread);
uploadResponse = await s3.UploadAsync(app, storage);
}
catch (Exception e)
{
app.LogMessage(file.FullName + " - " + e.Message);
errors++;
display = activeIndex.ToString() + " of " + fileCounts.ToString() + " ERROR";
}
try
{
if (uploadResponse)
{
if (existing == null)
{
documents.Insert(document);
display = activeIndex.ToString() + " of " + fileCounts.ToString() + " UPLOADED";
}
else
{
existing.DateModified = file.LastWriteTimeUtc;
documents.Update(existing);
display = activeIndex.ToString() + " of " + fileCounts.ToString() + " UPDATED";
}
uploaded++;
string _uploadedMsg = "Completed " + uploaded.ToString() + " of " + fileCounts.ToString();
uploadedMessage(_uploadedMsg);
string _skippedMsg = "Skipped " + skipped.ToString() + " of " + fileCounts.ToString();
skippedMessage(_skippedMsg);
}
}
catch (Exception e)
{
app.LogMessage(file.FullName + " - Error during LiteDB - " + e.Message);
errors++;
display = activeIndex.ToString() + " of " + fileCounts.ToString() + " ERROR";
}
message(display);
}
else
{
var display = activeIndex.ToString() + " of " + fileCounts.ToString() + " SKIPPED";
message(display);
skipped++;
string _uploadedMsg = "Completed " + uploaded.ToString() + " of " + fileCounts.ToString();
uploadedMessage(_uploadedMsg);
string _skippedMsg = "Skipped " + skipped.ToString() + " of " + fileCounts.ToString();
skippedMessage(_skippedMsg);
}
}
catch (Exception e)
{
app.LogMessage("Error during LiteDB - " + e.Message);
errors++;
}
}
}
if (threadId < threadsArray.Length)
{
completedThreads++;
message("FS: Thread " + threadId + " DONE");
string statusofThread = MakeStatusOfThread(threadId, 0, "Finished");
threadMessage(statusofThread);
if (completedThreads + 1 > ThreadsNum)
{
IsRunning = false;
message("FS: Skipped " + skipped.ToString());
message("FS: Uploaded " + uploaded.ToString());
message("FS: Done loading documents");
app.LogMessage("FS: Skipped " + skipped.ToString());
app.LogMessage("FS: Uploaded " + uploaded.ToString());
app.LogMessage("FS: Done loading documents");
handleUploadCompleted("completed");
if (uploaded > 0)
{
string endMessage = "{\"text\":\"End uploading\",\"blocks\":[{\"type\": \"section\",\"text\": {\"type\": \"mrkdwn\",\"text\": \"End uploading:\"}},{\"type\": \"section\",\"block_id\": \"section789\",\"fields\": [{\"type\": \"mrkdwn\",\"text\": \">*Bucket* : " + app.Settings.BucketName + "\n>*Uploaded* : " + uploaded.ToString() + "\n>*Skipped:* : " + skipped.ToString() + "\n*Errors:* : " + errors.ToString() + "\n>\"}]}]}";
await app.Slack.TryPostJsonAsync(endMessage);
}
}
}
});
threadsArray[i].Start(i);
}
return(true);
}