public bool ConnectionTest(AppRepository app)
{
var databaseName = app.Settings.MyobDatabaseName;
var serverName = app.Settings.MyobServerName;
if (!string.IsNullOrEmpty(databaseName) && !string.IsNullOrEmpty(serverName))
{
string connectionString = GetConnectionString(app);
var sqlcon = new SqlConnection(connectionString);
try
{
sqlcon.Open();
sqlcon.Close();
return(true);
}
catch (Exception e)
{
Console.WriteLine(e);
return(false);
}
}
return(false);
}
public bool ValidateMigrateKey(string migrateKey, AppRepository app)
{
bool result = false;
using (AmazonS3Client client = GetS3Client(app))
{
try
{
ListBucketsResponse response = client.ListBuckets();
foreach (S3Bucket b in response.Buckets)
{
Console.WriteLine("{0}\t{1}", b.BucketName, b.CreationDate);
if (b.BucketName == migrateKey)
{
result = true;
break;
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
return(result);
}
public async Task <bool> Export(AppRepository app, S3StorageRepository S3, Del message, Del threadMessage, Del uploadStatusMessage, Del uploadedMessage, Del skippedMessage, Del handleUploadCompleted)
{
IsRunning = true;
IsUploadingStop = false;
await LoadDocuments(app, message, threadMessage, uploadStatusMessage, uploadedMessage, skippedMessage, handleUploadCompleted);
return(false);
}
public async Task <bool> Export(AppRepository app, S3StorageRepository S3, Del message)
{
IsRunning = true;
LoadDocuments(app, message);
await ExportDocuments(app, S3, message);
IsRunning = false;
return(false);
}
private static string GetConnectionString(AppRepository app)
{
var databaseName = app.Settings.MyobDatabaseName;
var serverName = app.Settings.MyobServerName;
var connectionString = "SERVER =" + serverName + "; Trusted_Connection=true;INITIAL CATALOG=" + databaseName + "; Connection Timeout=30";
return(connectionString);
}
private async Task CloseAndUpload(AppRepository app, S3StorageRepository S3, Del message, StreamWriter jsonwrite, string file_path, string file_name)
{
if (jsonwrite != null)
{
jsonwrite.Close();
StorageRequest storage = new StorageRequest
{
Name = file_name,
Type = "import",
EncryptionKeyId = null,
Path = file_path,
};
message("Uploading " + file_name);
await S3.UploadAsync(app, storage);
}
}
private async Task ExportDocuments(AppRepository app, S3StorageRepository S3, Del message)
{
var documents = Database.GetCollection <Document>("myob_documents");
var count = 0;
var file_index = 0;
var file_path = "";
var file_name = "";
StreamWriter jsonwrite = null;
var records = documents.FindAll();
foreach (var record in records)
{
if (count % 50000 == 0)
{
await CloseAndUpload(app, S3, message, jsonwrite, file_path, file_name);
file_name = "index_" + file_index + ".json";
message("Creating export file " + file_name);
file_path = Environment.CurrentDirectory + "\\" + file_name;
FileStream fs = File.Create(file_path);
fs.Close();
file_index++;
jsonwrite = new StreamWriter(file_path, true);
}
string json = JsonConvert.SerializeObject(record, Formatting.None);
jsonwrite.WriteLine(json);
count++;
}
await CloseAndUpload(app, S3, message, jsonwrite, file_path, file_name);
}
private void LoadDocuments(AppRepository app, Del message)
{
string connectionString = GetConnectionString(app);
var sqlcon = new SqlConnection(connectionString);
var sqlcon2 = new SqlConnection(connectionString);
message("Getting documents remaining to be exported");
int DocumentCount = GetDocumentCount(sqlcon, app.Settings.LastDocumentId, app.Settings.LastDocumentModified);
var more = true;
var counter = 0;
var documents = Database.GetCollection <Document>("myob_documents");
documents.EnsureIndex(x => x.Reference_Number);
while (more)
{
more = false;
SqlCommand command = GetDocumentQuery(sqlcon, app.Settings.LastDocumentId, app.Settings.LastDocumentModified);
message("Refreshing dataset to export");
sqlcon.Open();
using (SqlDataReader reader = command.ExecuteReader())
{
try
{
while (reader.Read())
{
var document = new Document();
document.Import(reader);
GetDocumentCategories(sqlcon2, document);
app.Settings.LastDocumentId = document.DocumentId;
if (document.DateModified > app.Settings.LastDocumentModified || app.Settings.LastDocumentModified == null)
{
app.Settings.LastDocumentModified = document.DateModified;
}
counter++;
message("Collecting " + counter + " of " + DocumentCount.ToString());
more = true;
SaveDocument(documents, document);
app.Save();
}
}
catch (Exception e)
{
message(e.Message);
}
}
sqlcon.Close();
}
message("Done loading documents");
}
private async Task <bool> LoadDocuments(AppRepository app, Del message, Del threadMessage, Del uploadStatusMessage, Del uploadedMessage, Del skippedMessage, Del handleUploadCompleted)
{
app.LogMessage("FS: Getting documents remaining to be exported");
uploadStatusMessage("Getting documents remaining to be exported");
var fileList = new List <FileInfo>();
var folderList = new List <DirectoryInfo>();
foreach (string folder in app.Settings.FolderLists)
{
DirectoryInfo d = new DirectoryInfo(folder);
var r = d.GetFiles("*.*", SearchOption.AllDirectories);
fileList.AddRange(r);
uploadStatusMessage("Loading subfolders " + d.Name);
for (int i = 0; i < r.Count(); i++)
{
folderList.Add(d);
}
}
FileInfo[] Files = fileList.ToArray();
DirectoryInfo[] Dirs = folderList.ToArray();
var fileIndex = 0;
var completedThreads = 0;
var fileCounts = Files.Count();
var skipped = 0;
var uploaded = 0;
var errors = 0;
var documents = Database.GetCollection <Document>("documents");
documents.EnsureIndex(x => x.FilePath);
ThreadsNum = ThreadsNum < fileCounts ? ThreadsNum : fileCounts;
Thread[] threadsArray = new Thread[ThreadsNum];
for (int i = 0; i < ThreadsNum; i++)
{
threadsArray[i] = new Thread(async(object param) =>
{
int threadId = (int)param;
while (fileIndex < fileCounts)
{
if (IsUploadingStop)
{
Thread.Sleep(1000);
}
else
{
try
{
var activeIndex = fileIndex;
fileIndex++;
if (activeIndex < ThreadsNum)
{
activeIndex = threadId;
}
var file = Files[activeIndex];
Document document = new Document();
document.FilePath = file.FullName;
document.DateModified = file.LastWriteTimeUtc;
uploadStatusMessage(GetWorkingFolder(Dirs[activeIndex].ToString()));
var existing = documents.FindOne(x => x.FilePath == document.FilePath);
StorageRequest storage = new StorageRequest
{
Path = file.FullName,
Name = file.Name,
Type = makeCloudFolderPath(Dirs[activeIndex].ToString(), file.Name, file.FullName, app.Settings.IncludeFolderName),
EncryptionKeyId = null,
};
S3StorageRepository s3 = new S3StorageRepository(Database);
if (existing == null || existing.DateModified < document.DateModified)
{
var display = "";
var uploadResponse = false;
try
{
string statusofThread = MakeStatusOfThread(threadId, file.Length, file.Name);
threadMessage(statusofThread);
uploadResponse = await s3.UploadAsync(app, storage);
}
catch (Exception e)
{
app.LogMessage(file.FullName + " - " + e.Message);
errors++;
display = activeIndex.ToString() + " of " + fileCounts.ToString() + " ERROR";
}
try
{
if (uploadResponse)
{
if (existing == null)
{
documents.Insert(document);
display = activeIndex.ToString() + " of " + fileCounts.ToString() + " UPLOADED";
}
else
{
existing.DateModified = file.LastWriteTimeUtc;
documents.Update(existing);
display = activeIndex.ToString() + " of " + fileCounts.ToString() + " UPDATED";
}
uploaded++;
string _uploadedMsg = "Completed " + uploaded.ToString() + " of " + fileCounts.ToString();
uploadedMessage(_uploadedMsg);
string _skippedMsg = "Skipped " + skipped.ToString() + " of " + fileCounts.ToString();
skippedMessage(_skippedMsg);
}
}
catch (Exception e)
{
app.LogMessage(file.FullName + " - Error during LiteDB - " + e.Message);
errors++;
display = activeIndex.ToString() + " of " + fileCounts.ToString() + " ERROR";
}
message(display);
}
else
{
var display = activeIndex.ToString() + " of " + fileCounts.ToString() + " SKIPPED";
message(display);
skipped++;
string _uploadedMsg = "Completed " + uploaded.ToString() + " of " + fileCounts.ToString();
uploadedMessage(_uploadedMsg);
string _skippedMsg = "Skipped " + skipped.ToString() + " of " + fileCounts.ToString();
skippedMessage(_skippedMsg);
}
}
catch (Exception e)
{
app.LogMessage("Error during LiteDB - " + e.Message);
errors++;
}
}
}
if (threadId < threadsArray.Length)
{
completedThreads++;
message("FS: Thread " + threadId + " DONE");
string statusofThread = MakeStatusOfThread(threadId, 0, "Finished");
threadMessage(statusofThread);
if (completedThreads + 1 > ThreadsNum)
{
IsRunning = false;
message("FS: Skipped " + skipped.ToString());
message("FS: Uploaded " + uploaded.ToString());
message("FS: Done loading documents");
app.LogMessage("FS: Skipped " + skipped.ToString());
app.LogMessage("FS: Uploaded " + uploaded.ToString());
app.LogMessage("FS: Done loading documents");
handleUploadCompleted("completed");
if (uploaded > 0)
{
string endMessage = "{\"text\":\"End uploading\",\"blocks\":[{\"type\": \"section\",\"text\": {\"type\": \"mrkdwn\",\"text\": \"End uploading:\"}},{\"type\": \"section\",\"block_id\": \"section789\",\"fields\": [{\"type\": \"mrkdwn\",\"text\": \">*Bucket* : " + app.Settings.BucketName + "\n>*Uploaded* : " + uploaded.ToString() + "\n>*Skipped:* : " + skipped.ToString() + "\n*Errors:* : " + errors.ToString() + "\n>\"}]}]}";
await app.Slack.TryPostJsonAsync(endMessage);
}
}
}
});
threadsArray[i].Start(i);
}
return(true);
}
public AppConfigRepository()
{
Pile = new AppRepository(Database, slack);
}
public async Task <bool> UploadAsync(AppRepository app, StorageRequest storageRequest)
{
using (AmazonS3Client client = GetS3Client(app))
{
try
{
string bucket = app.Settings.BucketName;
if (string.IsNullOrEmpty(bucket))
{
throw new Exception("Application settings incomplete");
}
StorageCredentials credentials = new StorageCredentials()
{
BucketName = bucket,
};
PutObjectRequest putRequest = new PutObjectRequest
{
BucketName = $"{credentials.BucketName}",
Key = $"{storageRequest.Type}/{storageRequest.Name}",
//ServerSideEncryptionMethod = ServerSideEncryptionMethod.AWSKMS,
};
if (storageRequest.InputStream != null)
{
putRequest.InputStream = storageRequest.InputStream;
}
else if (!string.IsNullOrEmpty(storageRequest.Path))
{
putRequest.FilePath = storageRequest.Path;
}
else
{
putRequest.ContentBody = storageRequest.Content;
}
if (string.IsNullOrEmpty(putRequest.ContentBody) && string.IsNullOrEmpty(putRequest.FilePath) && putRequest.InputStream == null)
{
return(false);
}
if (!string.IsNullOrEmpty(storageRequest.EncryptionKeyId))
{
putRequest.ServerSideEncryptionKeyManagementServiceKeyId = storageRequest.EncryptionKeyId;
}
var response = await client.PutObjectAsync(putRequest);
return(true);
}
catch (AmazonS3Exception amazonS3Exception)
{
if (amazonS3Exception.ErrorCode != null && (amazonS3Exception.ErrorCode.Equals("InvalidAccessKeyId") || amazonS3Exception.ErrorCode.Equals("InvalidSecurity")))
{
throw new Exception("Check the provided AWS Credentials.");
}
throw new Exception("Error occurred: " + amazonS3Exception.Message);
}
catch (Exception e)
{
throw new Exception("Error occurred: " + e.Message);
}
}
}
private AmazonS3Client GetS3Client(AppRepository app)
{
// ... some code
}