/// <summary>
/// fill an expression with data from Sharepoint
/// </summary>
/// <param name="dataSchema">string with $$XX$$ to represent data to replace</param>
/// <param name="item">record in sharepoint </param>
/// <returns></returns>
public string TransformSharepointText(string dataSchema, ListItem item)
{
int startIndex, endIndex;
string columnName;
while (dataSchema.Contains("$$"))
{
startIndex = dataSchema.IndexOf("$$");
endIndex = dataSchema.IndexOf("$$", startIndex + 2);
columnName = dataSchema.Substring(startIndex + 2, (endIndex - (startIndex + 2)));
dataSchema = dataSchema.Remove(startIndex, (endIndex - startIndex) + 2);
try
{
if (item[columnName] != null)
{
dataSchema = dataSchema.Insert(startIndex, item[columnName].ToString());
}
}
catch (Exception ex)
{
TOOLS.LogErrorAndExitProgram($"Error occured when loading data : {columnName} from sharepoint please check your $$ syntaxe");
}
}
return(dataSchema);
}
/// <summary>
///
/// </summary>
/// <param name="exportType">the export type (only for log)</param>
/// <param name="url">url of http request</param>
/// <param name="periodicexport">periodic export config</param>
/// <param name="path">path</param>
/// <param name="postDataIds">array of ids to export</param>
/// <param name="fileNamePrefix">file name prefix</param>
internal async void RunPeriodicExport(string exportType, string url, PeriodicExport periodicexport, string path, string[] postDataIds, string fileNamePrefix)
{
if (!string.IsNullOrEmpty(path))
{
HttpResponseMessage response;
Log.Debug($"--Export to {exportType}");
response = await KfApiManager.HttpClient.PostAsJsonAsync($"{KfApiManager.KfApiUrl}/{url}", new { data_ids = postDataIds });
if (response.IsSuccessStatusCode)
{
string filePath = path;
string fileName = GetFileName(response, periodicexport.SpLibrary, Path.Combine(periodicexport.SpWebSiteUrl, filePath), fileNamePrefix);
using (var ms = new MemoryStream())
{
filePath = Path.Combine(periodicexport.LibraryName, filePath);
Log.Debug($"-----Downloading file : {fileName} from kizeoForms");
await response.Content.CopyToAsync(ms);
SendPeriodicExportToSpLibrary(ms, periodicexport.SpLibrary, filePath, fileName);
}
}
else
{
TOOLS.LogErrorAndExitProgram($"Error loading the export : {exportType} in path : {path}");
}
}
else
{
TOOLS.LogErrorAndExitProgram($"le champ path de l'export {exportType} est vide");
}
}
/// <summary>
/// Fill kizeo Forms External list with SharePoint data acording to the Config file
/// </summary>
/// <returns></returns>
public static async Task FillKfExtListsFromSp()
{
HttpResponseMessage response;
Log.Info("Updating KF's external lists from Sharepoint's lists");
foreach (var spListToExtList in Config.SpListsToExtLists)
{
Log.Debug($"Loading Sharepoint's list: {spListToExtList.SpListId}");
var spList = SpManager.LoadSpList(spListToExtList.SpListId);
ListItemCollection items = SpManager.LoadListItems(spList);
Log.Debug("Sharepoint's list Succesfully Loaded");
Log.Debug($"Loading KF's extrenal list: {spListToExtList.ExListId}");
response = await HttpClient.GetAsync($"{Config.KizeoConfig.Url}/rest/v3/lists/{spListToExtList.ExListId}");
if (response.IsSuccessStatusCode)
{
GetExtListRespViewModel kfExtList = await response.Content.ReadAsAsync <GetExtListRespViewModel>();
if (kfExtList.ExternalList == null)
{
TOOLS.LogErrorAndExitProgram($"Can not find an externalList with for id {spListToExtList.ExListId}, please check if this is a valid id.");
}
Log.Debug("KF's list successfully loaded");
List <string> linesToAdd = new List <string>();
foreach (var item in items)
{
Log.Debug($"Processing item : {item["Title"]}");
linesToAdd.Add(SpManager.TransformSharepointText(spListToExtList.DataSchema, item));
}
response = await HttpClient.PutAsJsonAsync($"{Config.KizeoConfig.Url}/rest/v3/lists/{spListToExtList.ExListId}", new { items = linesToAdd });
if (!response.IsSuccessStatusCode)
{
Log.Error("Error when updating external list with data :");
foreach (var line in linesToAdd)
{
TOOLS.LogErrorwithoutExitProgram("\n" + line);
}
TOOLS.LogErrorAndExitProgram($"Error when updating external list {spListToExtList.ExListId} ");
}
else
{
Log.Info($"External list {spListToExtList.ExListId} updated successfully");
}
}
}
}
/// <summary>
/// load and parse the config file
/// </summary>
/// <param name="ConfFilePath">path of the config file</param>
/// <returns>instance of Config that holds the current config</returns>
public static Config GetConfig(string ConfFilePath)
{
string path = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), "Kizeo");
string filePath = Path.Combine(path, ConfFilePath);
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
if (!System.IO.File.Exists(filePath))
{
using (FileStream fs = System.IO.File.Create(filePath))
{
}
}
try
{
String jsonText = new StreamReader(filePath).ReadToEnd();
Config = JsonConvert.DeserializeObject <Config>(jsonText);
}
catch (FileNotFoundException ex)
{
TOOLS.LogErrorAndExitProgram("file not found " + ex.Message);
}
catch (DirectoryNotFoundException ex)
{
TOOLS.LogErrorAndExitProgram("File not found" + ex.Message);
}
catch (ArgumentNullException ex)
{
TOOLS.LogErrorAndExitProgram("no configFile was given" + ex.Message);
}
catch (JsonReaderException ex)
{
TOOLS.LogErrorAndExitProgram("Error parsing json config" + ex.Message);
}
catch (JsonSerializationException ex)
{
TOOLS.LogErrorAndExitProgram("Erreur lors de la transforamtion du fichier config en objet :" + ex.Message);
}
catch (Exception ex)
{
TOOLS.LogErrorAndExitProgram(ex.Message);
}
return(Config);
}
/// <summary>
/// transform an expretion with expretion + data
/// </summary>
/// <param name="formId">the formId</param>
/// <param name="dataId">the DataId</param>
/// <param name="columnSelector">expresion to replace into it</param>
/// <returns></returns>
public async Task <string[]> TransformText(string formId, string dataId, string columnSelector)
{
if (string.IsNullOrEmpty(columnSelector))
{
TOOLS.LogErrorAndExitProgram("Path = null " + columnSelector + " 3");
return(null);
}
else
{
HttpResponseMessage response = await HttpClient.PostAsJsonAsync($"{KfApiUrl}/rest/v3/forms/{formId}/transformText",
new { textToTransform = columnSelector, data_ids = new string[] { dataId } });
TransformTextRespViewModel transformedText = await response.Content.ReadAsAsync <TransformTextRespViewModel>();
return(transformedText.TextDatas.Where(td => td.Data_id == dataId).First().Text);
}
}
/// <summary>
/// Run an export : download the file and send it to sharePoint
/// </summary>
/// <param name="exportType"> Export type used only for logging </param>
/// <param name="url"> Url of the request that is going to be executed </param>
/// <param name="isToExport"> Boolean to export or not</param>
/// <param name="formToSpLibrary"> Form to sp library to get config data </param>
/// <param name="data"> The data that holds the dataId and the formId</param>
/// <param name="path"> the kizeoforms expresion ## ## that controls where the file will be sent </param>
/// <param name="postDataIds">Ids of the data that is going to be exported</param>
public async void RunExport(string exportType, string url, bool isToExport, FormToSpLibrary formToSpLibrary, FormData data, string path, string[] postDataIds = null)
{
if (isToExport)
{
if (!string.IsNullOrEmpty(path))
{
HttpResponseMessage response;
Log.Debug($"--Export to {exportType}");
if (postDataIds == null)
{
response = await KfApiManager.HttpClient.GetAsync($"{KfApiManager.KfApiUrl}/{url}");
}
else
{
response = await KfApiManager.HttpClient.PostAsJsonAsync($"{KfApiManager.KfApiUrl}/{url}", new { data_ids = postDataIds });
}
if (response.IsSuccessStatusCode)
{
string filePath = (await KfApiManager.TransformText(data.FormID, data.Id, path)).First();
string fileName = GetFileName(response, formToSpLibrary.SpLibrary, Path.Combine(formToSpLibrary.SpWebSiteUrl, filePath), "");
using (var ms = new MemoryStream())
{
filePath = Path.Combine(formToSpLibrary.LibraryName, filePath);
Log.Debug($"-----Downloading file : {fileName} from kizeoForms");
await response.Content.CopyToAsync(ms);
SendToSpLibrary(ms, formToSpLibrary.SpLibrary, formToSpLibrary.MetaData, data, filePath, fileName);
}
}
else
{
TOOLS.LogErrorAndExitProgram($"Error loading the export : {exportType} in path : {path}");
}
}
else
{
TOOLS.LogErrorAndExitProgram($"le champ path de l'export {exportType} est vide");
}
}
}
/// <summary>
/// get all libraries folders paths
/// </summary>
/// <param name="spLibrary">the guid of the SpLibrary</param>
/// <param name="spWebSiteUrl">SharePoint webSite url</param>
/// <returns></returns>
public List <string> GetAllLibraryFolders(List spLibrary, string spWebSiteUrl)
{
var folderItems = spLibrary.GetItems(CamlQuery.CreateAllFoldersQuery());
lock (locky)
{
try
{
Context.Load(folderItems, f => f.Include(i => i.Folder));
Context.ExecuteQuery();
}
catch (Exception ex)
{
TOOLS.LogErrorAndExitProgram($"Error while retriving All folders from library : {spLibrary.Id} \n {ex.Message}");
}
return(folderItems.Select(i => i.Folder.ServerRelativeUrl.Replace(spWebSiteUrl + "/", "")).ToList());
}
}
/// <summary>
/// Load sharepoint lite records
/// </summary>
/// <param name="spList">guid of the spList</param>
/// <returns> a listItem collection</returns>
public ListItemCollection LoadListItems(List spList)
{
try
{
lock (locky)
{
ListItemCollection items = spList.GetItems(new CamlQuery());
Context.Load(items);
Context.ExecuteQuery();
return(items);
}
}
catch (Exception ex)
{
TOOLS.LogErrorAndExitProgram($"Error while retrieving items from List {spList.Id} {ex.Message}");
}
return(null);
}
/// <summary>
/// Load sharepoint lsit
/// </summary>
/// <param name="listId">guid of the SharePoint list</param>
/// <returns></returns>
public List LoadSpList(Guid listId)
{
var spList = Context.Web.Lists.GetById(listId);
Context.Load(spList);
try
{
lock (locky)
{
Context.ExecuteQuery();
return(spList);
}
}
catch (Exception ex)
{
TOOLS.LogErrorAndExitProgram($"Error while loading Sharepoint list {listId} " + ex.Message);
Application.Restart();
Environment.Exit(0);
return(null);
}
}
/// <summary>
/// return all sharepoint files
/// </summary>
/// <param name="spLibrary">guid of sharepoint library</param>
/// <param name="path">the path in the spLibrary where the paths would be retrieved</param>
/// <returns></returns>
public List <string> GetSpFiles(List spLibrary, string path)
{
try
{
var query = CamlQuery.CreateAllItemsQuery();
query.FolderServerRelativeUrl = path;
lock (locky)
{
var fileItems = spLibrary.GetItems(query);
Context.Load(fileItems, x => x.Include(xx => xx.File.Name));
Context.Load(fileItems, x => x.Include(xx => xx.ContentType.Name));
Context.ExecuteQuery();
return(fileItems.Where(f => f.ContentType.Name != "Folder").Select(s => s.File.Name).ToList());
}
}
catch (Exception EX)
{
TOOLS.LogErrorAndExitProgram($"Can not get files from {path} to check if the current file already exist.\n {EX.Message}");
return(null);
}
}
/// <summary>
/// Create instance of Shareoint manager and initialise the context
/// </summary>
/// <param name="spUrl"> SharePoint url</param>
/// <param name="spUser">UserName</param>
/// <param name="spPwd">Password</param>
///
public SharePointManager(string spDomain, string spClientId, string spClientSecret, KizeoFormsApiManager kfApiManager_)
{
KfApiManager = kfApiManager_;
Log.Debug($"Configuring Sharepoint Context");
locky = new object();
lockyFileName = new object();
try
{
Context = new AuthenticationManager().GetAppOnlyAuthenticatedContext(spDomain, spClientId, spClientSecret);
var web = Context.Web;
lock (locky)
{
Context.Load(web);
Context.ExecuteQuery();
}
Log.Debug($"Configuration succeeded");
}
catch (Exception ex)
{
TOOLS.LogErrorAndExitProgram("Error occured while initializing sharepoint config : " + ex.Message);
}
}
public async Task <string[]> TransformTextAddItem(string formId, string dataId, string columnSelector)
{
if (string.IsNullOrEmpty(columnSelector))
{
TOOLS.LogErrorAndExitProgram("Path = null " + columnSelector + " 3");
return(null);
}
else
{
HttpResponseMessage response = await HttpClient.PostAsJsonAsync($"{KfApiUrl}/rest/v3/forms/{formId}/transformText",
new { textToTransform = columnSelector, data_ids = new string[] { dataId } });
TransformTextRespViewModel transformedText = await response.Content.ReadAsAsync <TransformTextRespViewModel>();
/* string columnValue = transformedText.TextDatas.Where(td => td.Data_id == dataId).First().Text.First();*/
string[] columnValue = transformedText.TextDatas.Where(td => td.Data_id == dataId).First().Text;
/* if (columnValue.Contains("##"))
* TOOLS.LogErrorAndExitProgram($"No column name found in kizeo forms acording to the expression : {columnValue}");
*/
return(columnValue);
}
}
/// <summary>
/// Create an instance of KFAPiManager
/// </summary>
/// <param name="baseUrl">url of kizeo Forms</param>
/// <param name="token">the token to authentificate</param>
public KizeoFormsApiManager(string baseUrl, string token)
{
HttpClient = new HttpClient();
KfApiUrl = baseUrl;
Log.Debug($"Configuring Http Client");
try
{
HttpClient.BaseAddress = new Uri(baseUrl);
HttpClient.DefaultRequestHeaders.Accept.Clear();
HttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
if (string.IsNullOrEmpty(token))
{
throw new ArgumentException("Kizeo forms authentification token can not be null");
}
HttpClient.DefaultRequestHeaders.TryAddWithoutValidation("Authorization", token);
TestKfApi(baseUrl).Wait();
Log.Debug($"Configuration succeeded");
}
catch (ArgumentNullException ex)
{
TOOLS.LogErrorAndExitProgram("url kizeo forms can not be empty" + ex.Message);
}
catch (HttpRequestException)
{
TOOLS.LogErrorAndExitProgram("Can't connect to Kizeo forms server");
}
catch (Exception ex)
{
TOOLS.LogErrorAndExitProgram("Error while trying access the server : " + ex.Message);
}
}
/// <summary>
/// Load sp library
/// </summary>
/// <param name="periodicExport">periodic export config</param>
/// <returns></returns>
public List LoadSpLibrary(PeriodicExport periodicExport)
{
var spLibrary = Context.Web.Lists.GetById(periodicExport.SpLibraryId);
Context.Load(spLibrary);
Context.Load(spLibrary, spl => spl.Title);
lock (locky)
{
try
{
Context.ExecuteQuery();
periodicExport.SpWebSiteUrl = ($@"{spLibrary.ParentWebUrl}/{spLibrary.Title}");
periodicExport.LibraryName = spLibrary.Title;
Log.Debug("-SharePoint library Succesfully Loaded");
return(spLibrary);
}
catch (Exception ex)
{
TOOLS.LogErrorAndExitProgram("Error while loading Sharepoint Library " + ex.Message);
return(null);
}
}
}
static void Main(string[] args)
{
System.Globalization.CultureInfo.DefaultThreadCurrentUICulture = System.Globalization.CultureInfo.GetCultureInfo("en-US");
// Please don't choose too short time
int kfToSpsyncTime = 5;
int spToKfSyncTime = 30;
try
{
XmlConfigurator.Configure();
Log.Info("Configuration");
Log.Debug($"Get and serialize config file from : {ConfFile}.");
Config = GetConfig(ConfFile);
Log.Debug($"Configuration succeeded");
Log.Info($"The application will automatically restart in 12 hours to renew SharePoint's access token.");
Thread t = new Thread(
() =>
{
Thread.Sleep(3600 * 12 * 1000);
Application.Restart();
Environment.Exit(0);
}
);
t.Start();
if (Config == null || Config.KizeoConfig == null || Config.SharepointConfig == null)
{
TOOLS.LogErrorAndExitProgram("The config file is empty or some data is missing.");
}
KfApiManager = new KizeoFormsApiManager(Config.KizeoConfig.Url, Config.KizeoConfig.Token);
HttpClient = KfApiManager.HttpClient;
SpManager = new SharePointManager(Config.SharepointConfig.SPDomain, Config.SharepointConfig.SPClientId, Config.SharepointConfig.SPClientSecret, KfApiManager);
Context = SpManager.Context;
FillKfExtListsFromSp().Wait();
FillSpListsFromKfData().Wait();
UploadExportsToSpLibraries().Wait();
initTimers(kfToSpsyncTime, spToKfSyncTime);
Log.Info($"Synchronisation will be executed every {kfToSpsyncTime} minutes.");
Console.ReadKey();
}
catch (Exception EX)
{
Log.Fatal(EX);
}
finally
{
Context.Dispose();
}
while (true)
{
Console.ReadKey();
}
}
/// <summary>
/// upload repports to sharepoint library acording to config file
/// </summary>
/// <returns></returns>
public static async Task UploadExportsToSpLibraries()
{
Log.Info($"Uploading exports to SharePoint's library");
FormDatasRespViewModel formData = null;
HttpResponseMessage response;
MarkDataReqViewModel dataToMark;
string formId;
foreach (var formToSpLibrary in Config.FormsToSpLibraries)
{
string marker = KfApiManager.CreateKFMarker(formToSpLibrary.FormId, formToSpLibrary.SpLibraryId);
formId = formToSpLibrary.FormId;
dataToMark = new MarkDataReqViewModel();
Log.Debug($"-Processing form : {formId}");
response = await HttpClient.GetAsync($"{Config.KizeoConfig.Url}/rest/v3/forms/{formId}/data/unread/{marker}/50?includeupdated");
if (response.IsSuccessStatusCode)
{
Log.Debug($"-Loading Sharepoint's library");
formToSpLibrary.SpLibrary = SpManager.LoadSpLibrary(formToSpLibrary);
formData = await response.Content.ReadAsAsync <FormDatasRespViewModel>();
if (formData.Data == null)
{
TOOLS.LogErrorAndExitProgram($"Can not find a form for id {formId}, check if this is a valid id.");
}
Log.Debug($"{formData.Data.Count} Data retrieved successfully from form");
var allSpPaths = SpManager.GetAllLibraryFolders(formToSpLibrary.SpLibrary, formToSpLibrary.SpWebSiteUrl);
foreach (var data in formData.Data)
{
Log.Debug($"-Processing data : {data.Id}");
var allExportPaths = await GetAllExportsPath(data, formToSpLibrary);
Log.Warn("Creating All Folders hierarchy");
foreach (var path in allExportPaths)
{
if (!string.IsNullOrEmpty(path) && !string.IsNullOrWhiteSpace(path) && !allSpPaths.Contains(path))
{
Log.Warn($"Creating path : {path}");
TOOLS.CreateSpPath(Context, path, formToSpLibrary.SpLibrary);
allSpPaths.Add(path);
}
}
Log.Debug($"--Processing data : {data.Id}");
SpManager.RunExport("PDF", $"rest/v3/forms/{formId}/data/{data.Id}/pdf", formToSpLibrary.ToStandardPdf, formToSpLibrary, data, formToSpLibrary.StandardPdfPath);
SpManager.RunExport("Excel", $"rest/v3/forms/{formId}/data/multiple/excel", formToSpLibrary.ToExcelList, formToSpLibrary, data, formToSpLibrary.ExcelListPath, new string[] { data.Id });
SpManager.RunExport("CSV", $"rest/v3/forms/{formId}/data/multiple/csv", formToSpLibrary.ToCsv, formToSpLibrary, data, formToSpLibrary.CsvPath, new string[] { data.Id });
SpManager.RunExport("CSV_Custom", $"rest/v3/forms/{formId}/data/multiple/csv_custom", formToSpLibrary.ToCsvCustom, formToSpLibrary, data, formToSpLibrary.CsvCustomPath, new string[] { data.Id });
SpManager.RunExport("Excel_Custom", $"rest/v3/forms/{formId}/data/multiple/excel_custom", formToSpLibrary.ToExcelListCustom, formToSpLibrary, data, formToSpLibrary.ExcelListCustomPath, new string[] { data.Id });
if (formToSpLibrary.Exports != null && formToSpLibrary.Exports.Count > 0)
{
foreach (var export in formToSpLibrary.Exports)
{
Log.Debug($"---Processing export : {export.Id}");
SpManager.RunExport("Initial Type", $"rest/v3/forms/{formId}/data/{data.Id}/exports/{export.Id}", export.ToInitialType, formToSpLibrary, data, export.initialTypePath);
SpManager.RunExport("Pdf Type", $"rest/v3/forms/{formId}/data/{data.Id}/exports/{export.Id}/pdf", export.ToPdf, formToSpLibrary, data, export.PdfPath);
}
}
dataToMark.Ids.Add(data.Id);
}
if (dataToMark.Ids.Count > 0)
{
response = await HttpClient.PostAsJsonAsync($"{Config.KizeoConfig.Url}/rest/v3/forms/{formId}/markasreadbyaction/{marker}", dataToMark);
Log.Debug($"-{dataToMark.Ids.Count} data marked");
}
}
}
}
/// <summary>
/// Fill Sharepoint list from kizeo forms data acording to the config file
/// </summary>
/// <returns></returns>
public static async Task FillSpListsFromKfData()
{
Log.Info("Filling Sharepoint's lists from KF's data");
FormDatasRespViewModel formData = null;
HttpResponseMessage response;
MarkDataReqViewModel dataToMark = new MarkDataReqViewModel();
foreach (var formToSpList in Config.FormsToSpLists)
{
string marker = KfApiManager.CreateKFMarker(formToSpList.FormId, formToSpList.SpListId);
string formId = formToSpList.FormId;
Log.Debug($"Processing form : {formId}");
if (formToSpList.DataMapping != null)
{
response = await HttpClient.GetAsync($"{Config.KizeoConfig.Url}/rest/v3/forms/{formId}/data/unread/{marker}/50?includeupdated");
if (response.IsSuccessStatusCode)
{
formData = await response.Content.ReadAsAsync <FormDatasRespViewModel>();
if (formData.Data == null)
{
TOOLS.LogErrorAndExitProgram($"Can not find a form with for id {formId}, please check if this is a valid id.");
}
Log.Debug($"{formData.Data.Count} data retrieved successfully from form.");
Log.Debug("Loading Sharepoint's list");
var spList = SpManager.LoadSpList(formToSpList.SpListId);
ListItemCollection allItems = SpManager.getAllListItems(spList);
Log.Debug("Sharepoint's list succesfully loaded");
dataToMark = new MarkDataReqViewModel();
foreach (var data in formData.Data)
{
try
{
var uniqueColumns = formToSpList.DataMapping.Where(dm => dm.SpecialType == "Unique").ToList();
await SpManager.AddItemToList(spList, formToSpList.DataMapping, data, dataToMark, uniqueColumns, allItems);
}
catch (ServerException ex)
{
TOOLS.LogErrorwithoutExitProgram($"Error while sending item {data.Id} from form {data.FormID} to the Sharepoint's list {spList.Id} : " + ex.Message);
}
}
if (dataToMark.Ids.Count > 0)
{
response = await HttpClient.PostAsJsonAsync($"{Config.KizeoConfig.Url}/rest/v3/forms/{formId}/markasreadbyaction/{marker}", dataToMark);
Log.Debug($"{dataToMark.Ids.Count} data marked");
}
}
}
else
{
TOOLS.LogErrorAndExitProgram("No datamapping was configured, please add a datamapping");
}
}
}