private void GetResultPage(ImportResponse response, int pageSize)
{
CachedImportRequest cachedRequest = this.GetFromCache(response.Context);
int itemCount = 0;
response.Objects = new List <CSEntryChange>();
response.TotalItems = cachedRequest.Count;
if (pageSize > 0)
{
CSEntryChange csentry;
while (cachedRequest.Queue.TryDequeue(out csentry))
{
itemCount++;
response.Objects.Add(csentry);
if (itemCount >= pageSize)
{
break;
}
}
}
if (cachedRequest.Queue.IsEmpty && cachedRequest.ProducerComplete)
{
response.HasMoreItems = false;
}
else
{
response.HasMoreItems = true;
}
}
private async Task <IResponse> DataBlockAppendAsync(IRequestContext ctx, Route route)
{
var tableName = GetAndValidateTableName(route);
var table = this.Database[tableName];
if (table == null)
{
return(ArribaResponse.BadRequest("Table {0} is not loaded or does not exist", tableName));
}
using (ctx.Monitor(MonitorEventLevel.Information, "Import.DataBlock", type: "Table", identity: tableName))
{
DataBlock block = await ctx.Request.ReadBodyAsync <DataBlock>();
table.AddOrUpdate(block, new AddOrUpdateOptions()
{
AddMissingColumns = true
});
ImportResponse response = new ImportResponse();
response.TableName = tableName;
response.Columns = block.Columns.Select((cd) => cd.Name).ToArray();
response.RowCount = block.RowCount;
return(ArribaResponse.Ok(response));
}
}
public async Task <ImportResponse> ImportCsv()
{
if (!Request.Content.IsMimeMultipartContent())
{
throw new HttpResponseException(HttpStatusCode.UnsupportedMediaType);
}
ImportResponse response = new ImportResponse();
var provider = new MultipartMemoryStreamProvider();
await Request.Content.ReadAsMultipartAsync(provider);
var file = provider.Contents.FirstOrDefault();
if (file != null)
{
response.Filename = file.Headers.ContentDisposition.FileName.Trim('\"');
string csv = await file.ReadAsStringAsync();
var csvService = new CsvService(ApplicationContext);
response = csvService.ImportCsv(csv);
}
return(response);
}
public async Task <ImportResponse> ImportGamesAsync(GameImportRequest request)
{
ImportResponse response = await CheckRequest(request.UserID, request.Games.Count);
if (!response.Successful)
{
return(response);
}
var user = await _userManager.FindByIdAsync(request.UserID);
foreach (var game in request.Games)
{
try
{
game.UserID = request.UserID;
game.UserNum = user.UserNum;
game.ID = 0;
_gameService.Add(game);
response.Imported++;
}
catch (Exception ex)
{
response.Failed++;
response.Message += $"{game.Title} - {ex.Message} : {ex.InnerException?.Message}";
}
}
response.Successful = true;
return(response);
}
/// <summary>
/// 导入客户数据
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
public JsonResult ImportCusData(string data)
{
string jsonStr = string.Empty;
bool result = false;
string retmsg = string.Empty;
LogicBiz biz = new LogicBiz();
RetMsg msg = biz.DataImport(customerUrl, data, SessionId);
if (!msg.IsSysError)
{
ImportResponse <CustomerErrorDetail> response = DataJsonSerializer <ImportResponse <CustomerErrorDetail> > .JsonToEntity(msg.Message);
if (response.Data.FalseCount == 0)
{
result = true; //导入成功
}
else
{
jsonStr = JsonConvert.SerializeObject(response.Data);
}
}
else
{
retmsg = msg.Message;
}
return(Json(new { Result = result, Msg = retmsg, Data = jsonStr }, JsonRequestBehavior.AllowGet));
}
public async Task <ImportResponse> ImportAlbumsAsync(AlbumImportRequest request)
{
ImportResponse response = await CheckRequest(request.UserID, request.Albums.Count);
if (!response.Successful)
{
return(response);
}
var user = await _userManager.FindByIdAsync(request.UserID);
foreach (var album in request.Albums)
{
try
{
album.UserID = request.UserID;
album.UserNum = user.UserNum;
album.ID = 0;
_albumService.Add(album);
response.Imported++;
}
catch (Exception ex)
{
response.Failed++;
response.Message += $"{album.Title}- {ex.Message} : {ex.InnerException?.Message} ,";
}
}
response.Successful = true;
return(response);
}
public async Task <ImportResponse> ImportBooksAsync(BookImportRequest request)
{
ImportResponse response = await CheckRequest(request.UserID, request.Books.Count);
if (!response.Successful)
{
return(response);
}
var user = await _userManager.FindByIdAsync(request.UserID);
foreach (var book in request.Books)
{
try
{
book.UserID = request.UserID;
book.UserNum = user.UserNum;
book.ID = 0;
_bookService.Add(book);
response.Imported++;
}
catch (Exception ex)
{
response.Failed++;
response.Message += $"{book.Title} - {ex.Message} : {ex.InnerException?.Message}";
}
}
response.Successful = true;
return(response);
}
public async Task <ImportResponse> ImportPopsAsync(PopImportRequest request)
{
ImportResponse response = await CheckRequest(request.UserID, request.Pops.Count);
if (!response.Successful)
{
return(response);
}
var user = await _userManager.FindByIdAsync(request.UserID);
foreach (var pop in request.Pops)
{
try
{
pop.UserID = request.UserID;
pop.UserNum = user.UserNum;
pop.ID = 0;
_popService.Add(pop);
response.Imported++;
}
catch (Exception ex)
{
response.Failed++;
response.Message += $"{pop.Title} - {ex.Message} : {ex.InnerException?.Message}";
}
}
response.Successful = true;
return(response);
}
private async Task <ImportResponse> CheckRequest(string userID, int count)
{
var response = new ImportResponse
{
NumRequested = count
};
if (string.IsNullOrWhiteSpace(userID))
{
response.Failed = count;
response.Message = "User ID Missing";
return(response);
}
var user = await _userManager.FindByIdAsync(userID);
if (!user.EnableImport)
{
response.Failed = count;
response.Message = "Import is not enabled for this user";
return(response);
}
response.Successful = true;
return(response);
}
/// <summary>
/// Handles the FinishButtonClick event of the ucWizard control.
/// </summary>
/// <param name="sender">The source of the event.</param>
/// <param name="e">The <see cref="System.Web.UI.WebControls.WizardNavigationEventArgs"/> instance containing the event data.</param>
void ucWizard_FinishButtonClick(object sender, WizardNavigationEventArgs e)
{
_ir = (ImportRequest)ViewState["_ir"];
if (_ir != null)
{
FillDefaultValues(dgMapping, null);
_ir = (ImportRequest)ViewState["_ir"];
ImportResponse irr = (ImportResponse)BusinessManager.Execute(_ir);
MappingError[] mas = irr.Errors;
if (mas.Length == 0)
{
lblResult.Text = GetGlobalResourceObject("IbnFramework.Common", "tImportWasSuccessfull").ToString();
}
else
{
ViewState["ErrorLog"] = MappingError.GetErrorLog(mas);
string sAction = String.Format("<a href=\"{1};\">{0}</a>",
GetGlobalResourceObject("IbnFramework.Common", "tErrorList").ToString(),
this.Page.ClientScript.GetPostBackClientHyperlink(lbErrorLog, String.Empty));
lblResult.Text = String.Format(
GetGlobalResourceObject("IbnFramework.Common", "tImportWithErrors").ToString(),
sAction);
}
string param = String.Empty;
if (!String.IsNullOrEmpty(_commandName))
{
param = (new CommandParameters(_commandName)).ToString();
}
Mediachase.Ibn.Web.UI.WebControls.CommandHandler.RegisterRefreshParentFromFrameScript(this.Page, param);
}
}
public IActionResult ImportFile(IFormFile file)
{
_logger.LogInfo("New file import request accepted");
if (file == null || file.Length == 0)
{
_logger.LogInfo("Invalid file (null or empty)");
return(BadRequest(new ImportResponse(null, "File is invalid.")));
}
string session = DateTime.Now.ToString("yyyyMMddHHmmss");
ImportStateService.SetInitialized(session);
ImportResponse resp = new ImportResponse(session, "OK");
//
string dir = _service.GetSavingDirectory() + session + "/";
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
string path = dir + file.FileName;
using (var fileStream = new FileStream(path, FileMode.Create)) {
file.CopyTo(fileStream);
}
//
Task.Run(() => ImportAsync(session, dir, file.FileName));
return(Ok(resp));
}
/// <summary>
/// Configures the import session at the beginning of an import
/// </summary>
/// <param name="configParameters">The configuration parameters supplied to this management agent</param>
/// <param name="types">The schema types that apply to this import run</param>
/// <param name="importRunStep">The definition of the current run step</param>
/// <returns>Results of the import setup</returns>
public OpenImportConnectionResults OpenImportConnection(KeyedCollection <string, ConfigParameter> configParameters, Schema types, OpenImportConnectionRunStep importRunStep)
{
try
{
this.suppliedConfigParameters = configParameters;
Logger.LogPath = this.LogPath;
Logger.WriteSeparatorLine('*');
Logger.WriteLine("Starting Import");
Logger.WriteLine("Import data from sync engine: " + importRunStep.CustomData);
this.client = new AcmaSyncServiceClient();
this.client.Open();
ImportStartRequest request = new ImportStartRequest();
request.ImportType = importRunStep.ImportType;
request.PageSize = 0;
request.Schema = types;
this.importResponse = this.client.ImportStart(request);
this.importPageSize = importRunStep.PageSize;
return(new OpenImportConnectionResults());
}
catch (Exception ex)
{
Logger.WriteException(ex);
throw;
}
}
public override global::System.Threading.Tasks.Task Import(Grpc.Core.IAsyncStreamReader <ImportRequest> requestStream, Grpc.Core.IServerStreamWriter <ImportResponse> responseStream, Grpc.Core.ServerCallContext context)
{
var res = new ImportResponse();
res.Result = "result123xyz";
Console.WriteLine("Server got request, sending response");
return(responseStream.WriteAsync(res));
}
public ImportResponse ImportPage(PageRequest request)
{
ImportResponse response = new ImportResponse();
response.Context = request.Context;
this.GetResultPage(response, request.PageSize);
return(response);
}
public IActionResult Import([FromForm] ImportRequest importRequest)
{
if (SecurityTokenMissing())
{
return(Unauthorized());
}
var documentId = Guid.NewGuid().ToString();
string import = GetMpwPath(documentId);
using (var fs = new FileStream(import, FileMode.Create))
{
importRequest.File.CopyTo(fs);
fs.Close();
}
var response = new ImportResponse
{
Header = new Header
{
Status = new Status {
Code = Status.SUCCESS
}
},
Body = new DocumentBody
{
Document = new Document
{
AppId = importRequest.AppId,
ApplicationName = "EditAreaTextParagraph",
BusDocId = importRequest.BusDocId,
CreationDate = DateTime.Now,
Deleted = false,
DocId = documentId,
DocTags = importRequest.DocTag,
DocumentVersion = "400101",
EditorVersion = "1.1.0.7948",
EngineVersion = "900101",
FileName = "EditAreaTextParagraph.mpw",
ImportDate = DateTime.Now,
LastSaveDate = DateTime.Now,
PackageFileName = "ExstreamPackage.pub",
PackageVersion = "900122",
PreviewPubFile = "PdfPreview.pub",
UserId = importRequest.OwnerId
}
}
};
return(new OkObjectResult(response));
}
/// <summary>
/// Imports genotype data from 23andMe service.
/// </summary>
public static void ImportGenotypeData(int profile_id)
{
//PUT /profiles/{profile_id}/genotype/import/23andme
ImportResponse res = ExecProc <ImportResponse>(
String.Format("profiles/{0}/genotype/import/23andme", profile_id),
new ImportRequest()
{
AccessToken = "ccea40d57c453af5d1d80daefd4036da",
ProfileId = "f4c9324568250f71",
RefreshToken = "f7c5da6f179e823a68b35b7da067e494",
},
"PUT");
}
private IResponse CsvAppend(IRequestContext ctx, Route route)
{
var content = ctx.Request.Headers["Content-Type"];
if (String.IsNullOrEmpty(content) || !String.Equals(content, "text/csv", StringComparison.OrdinalIgnoreCase))
{
return(ArribaResponse.BadRequest("Content-Type of {0} was not expected", content));
}
else if (!ctx.Request.HasBody)
{
return(ArribaResponse.BadRequest("Empty request body"));
}
var tableName = GetAndValidateTableName(route);
var table = this.Database[tableName];
if (table == null)
{
return(ArribaResponse.BadRequest("Table {0} is not loaded or does not exist", tableName));
}
var response = new ImportResponse();
response.TableName = tableName;
var config = new CsvReaderSettings()
{
DisposeStream = true, HasHeaders = true
};
var detail = new
{
RequestSize = ctx.Request.Headers["Content-Length"]
};
using (ctx.Monitor(MonitorEventLevel.Information, "Import.Csv", type: "Table", identity: tableName, detail: detail))
{
using (CsvReader reader = new CsvReader(ctx.Request.InputStream, config))
{
response.Columns = reader.ColumnNames;
foreach (var blockBatch in reader.ReadAsDataBlockBatch(BatchSize))
{
response.RowCount += blockBatch.RowCount;
table.AddOrUpdate(blockBatch);
}
}
}
return(ArribaResponse.Created(response));
}
public ImportResponse ImportStart(ImportStartRequest request)
{
Logger.WriteLine("Import request received");
MAStatistics.StartOperation(MAOperationType.Import);
ImportResponse response = new ImportResponse();
CachedImportRequest cachedRequest = new CachedImportRequest();
cachedRequest.Request = request;
if (cachedRequest.Request.ImportType == OperationType.Delta)
{
this.ProcessOperationEvents(AcmaEventOperationType.DeltaImport);
}
else
{
this.ProcessOperationEvents(AcmaEventOperationType.FullImport);
}
byte[] watermark = ActiveConfig.DB.GetHighWatermarkMAObjectsDelta();
response.Watermark = watermark == null ? null : Convert.ToBase64String(watermark);
cachedRequest.HighWatermark = watermark;
Logger.WriteLine("Got delta watermark: {0}", response.Watermark);
ResultEnumerator enumerator;
if (cachedRequest.Request.ImportType == OperationType.Delta)
{
enumerator = ActiveConfig.DB.EnumerateMAObjectsDelta(watermark);
}
else
{
enumerator = ActiveConfig.DB.EnumerateMAObjects(cachedRequest.Request.Schema.Types.Select(t => t.Name).ToList(), null, null);
}
cachedRequest.Queue = new ConcurrentQueue <CSEntryChange>();
cachedRequest.Count = enumerator.TotalCount;
this.StartProducerThread(enumerator, request.Schema, cachedRequest);
response.Context = Guid.NewGuid().ToString();
this.AddToCache(response.Context, cachedRequest);
this.GetResultPage(response, request.PageSize);
return(response);
}
private async Task <IResponse> ImportAsync(object request, string url)
{
using (var client = new HttpClient())
{
logger?.LogDebug($"Start request with base url {BaseUrl} and url {url}");
var urlBuilder = new StringBuilder();
client.BaseAddress = new Uri(BuildUrl(url));
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
client.DefaultRequestHeaders.Add("X-Token", Token);
logger?.LogDebug($"Endpoint Base-URL is: {client.BaseAddress.ToString()}");
var json = JsonConvert.SerializeObject(request);
var content = new StringContent(json, Encoding.UTF8, "application/json");
var response = await client.PostAsync(url, content);
var responseContent = await response.Content.ReadAsStringAsync();
try
{
if (response.IsSuccessStatusCode)
{
var importResponse = new ImportResponse((int)response.StatusCode, responseContent);
JsonConvert.PopulateObject(responseContent, importResponse);
logger?.LogInformation($"Request ({url}) successful: {importResponse.AddedCount} entities added, {importResponse.UpdatedCount} entities updated, {importResponse.RemovedCount} entities removed, {importResponse.IgnoredEntities.Count} entities ignored.");
return(importResponse);
}
else
{
var errorResponse = new ErrorResponse((int)response.StatusCode, responseContent);
JsonConvert.PopulateObject(responseContent, errorResponse);
logger?.LogError($"Request was not succesful: {errorResponse.Message}");
return(errorResponse);
}
}
catch (Exception e)
{
logger?.LogError("Failed to parse response.");
throw e;
}
}
}
private IResponse CsvAppend(IRequestContext ctx, Route route)
{
if (!ctx.Request.HasBody)
{
return(ArribaResponse.BadRequest("Empty request body"));
}
var tableName = GetAndValidateTableName(route);
var table = this.Database[tableName];
if (table == null)
{
return(ArribaResponse.BadRequest("Table {0} is not loaded or does not exist", tableName));
}
var response = new ImportResponse();
response.TableName = tableName;
var config = new CsvReaderSettings()
{
DisposeStream = true, HasHeaders = true
};
using (ctx.Monitor(MonitorEventLevel.Information, "Import.Csv", type: "Table", identity: tableName))
{
using (CsvReader reader = new CsvReader(ctx.Request.InputStream, config))
{
response.Columns = reader.ColumnNames;
foreach (var blockBatch in reader.ReadAsDataBlockBatch(BatchSize))
{
response.RowCount += blockBatch.RowCount;
table.AddOrUpdate(blockBatch, new AddOrUpdateOptions()
{
AddMissingColumns = true
});
}
}
}
return(ArribaResponse.Ok(response));
}
/// <summary>
/// Gets a batch of entries from the database and returns them to the synchronization service
/// </summary>
/// <param name="importRunStep">The current run step</param>
/// <returns>The results of the import batch</returns>
public GetImportEntriesResults GetImportEntries(GetImportEntriesRunStep importRunStep)
{
try
{
List <CSEntryChange> csentries = new List <CSEntryChange>();
PageRequest request = new PageRequest();
request.PageSize = this.importPageSize;
request.Context = this.importResponse.Context;
Logger.WriteLine("Requesting page of {0} results", LogLevel.Debug, this.importPageSize);
ImportResponse page = this.client.ImportPage(request);
Logger.WriteLine("Got page of {0} results", LogLevel.Debug, this.importPageSize);
foreach (CSEntryChange csentry in page.Objects)
{
csentries.Add(csentry);
if (csentry.ErrorCodeImport == MAImportError.ImportErrorCustomStopRun)
{
break;
}
if (string.IsNullOrWhiteSpace(csentry.ObjectType) || csentry.ErrorCodeImport != MAImportError.Success)
{
Logger.WriteLine("An error was detected in the following CSEntryChange object");
Logger.WriteLine(csentry.ToDetailString());
}
}
Logger.WriteLine("Returning page of {0} results. More to import: {1}", LogLevel.Debug, page.Objects.Count, page.HasMoreItems);
GetImportEntriesResults importReturnInfo = new GetImportEntriesResults();
importReturnInfo.MoreToImport = page.HasMoreItems;
importReturnInfo.CSEntries = csentries;
return(importReturnInfo);
}
catch (Exception ex)
{
Logger.WriteException(ex);
throw;
}
}
/// <summary>
/// Imports the CSV dictionary back into Umbraco
/// </summary>
/// <param name="csv">The CSV entries</param>
/// <returns></returns>
public ImportResponse ImportCsv(string csv)
{
ImportResponse response = new ImportResponse
{
Validation = this.ValidateCsv(csv)
};
if (!response.Validation.IsValid)
{
return(response);
}
var dtos = response.Validation.CsvRows.ToLanguageTextDtos();
var dictionaryService = new DictionaryDataService(appContext);
response.Update = dictionaryService.UpdateDtoItems(dtos);
return(response);
}
/// <summary>
///
/// </summary>
/// <param name="filePath"></param>
/// <returns></returns>
/// public ImportResponse ImportData(string listName, string fileName, string worksheet, string importType, string teamId)
public ImportResponse ImportData()
{
// get the web context for the response
WebOperationContext ctx = WebOperationContext.Current;
// set default status to 200-OK
ctx.OutgoingResponse.StatusCode = System.Net.HttpStatusCode.OK;
// initialize objects
ImportResponse response = new ImportResponse();
SPDataAccess spDataAccess = new SPDataAccess();
// create an instance of the worksheet and initialize SheetImport
SheetImport import = new SheetImport(new WorksheetUtilities(spDataAccess), spDataAccess);
try
{
MappingData mappingData = this.ValidateRequest(spDataAccess, true);
// import the file data using the uploaded file as input
response = import.ImportFileData(mappingData);
// check response status code - set to error if not 200
if (response.StatusCode != 200)
{
ctx.OutgoingResponse.StatusCode = System.Net.HttpStatusCode.InternalServerError;
}
}
catch (Exception ex)
{
ctx.OutgoingResponse.StatusCode = System.Net.HttpStatusCode.InternalServerError;
response.StatusCode = 520;
response.Message = "Error in Import.ImportData";
response.Errors = new List <ErrorObject>();
response.Errors.Add(new ErrorObject(ex.Message, ex.StackTrace));
}
return(response);
}
[HttpPost("import"), RequestSizeLimit(20 * 1024 * 1024)] // 20 MB
public async Task <ActionResult <ImportResponse> > Import([FromQuery] ImportArguments args)
{
string contentType = null;
string fileName = null;
Stream fileStream = null;
if (Request.Form.Files.Count > 0)
{
IFormFile formFile = Request.Form.Files[0];
contentType = formFile?.ContentType;
fileName = formFile?.FileName;
fileStream = formFile?.OpenReadStream();
}
try
{
var service = GetCrudService();
var result = await service.Import(fileStream, fileName, contentType, args);
var response = new ImportResponse
{
Inserted = result.Inserted,
Updated = result.Updated,
Milliseconds = result.Milliseconds
};
return(Ok(response));
}
finally
{
if (fileStream != null)
{
await fileStream.DisposeAsync();
}
}
}
/// <summary>
///
/// </summary>
/// <param name="mappingData"></param>
/// <returns></returns>
public ImportResponse ImportData(MappingData mappingData)
{
ImportStatus importStatus = new ImportStatus();
Dictionary <string, string> listMappings = new Dictionary <string, string>();
// initialize response
ImportResponse response = new ImportResponse();
response.Message = string.Empty;
// extract property values
Dictionary <string, string> mappings = mappingData.Mappings;
string fileName = mappingData.FileName;
string sheetName = mappingData.SheetName;
int rowCount = 0;
List <string> listColumns = new List <string>();
List <int> duplicateRows = new List <int>();
List <KPListItem> kpListItems = new List <KPListItem>();
// datatable contains column definitions and rows
DataTable dt = this.worksheetUtilities.GetDataTable(mappingData);
// initialize lookup list collections
this.domainManager.InitLookups();
// initialize entity list
// get entities by team and all sub-teams
// if team is supplied then use this for all rows in sheet
if (mappingData.TemplateType == ImportType.Export)
{
// these imports are for multiple teams so we need to get the team data
List <int> teamIds = GetTeamIds(dt, mappingData);
kpListItems = this.domainManager.GetUserEntities(mappingData.ListName, teamIds);
}
else
{
kpListItems = this.domainManager.GetUserEntities(mappingData.ListName, mappingData.TeamId.Value);
}
// get list columns from mapping
//List<string> kpListColumns = kpListItems[0].Keys.ToList();
List <string> kpListColumns = mappings.Keys.ToList();
// ensure that we have a valid sheet template by comparing the sheet columns to our mapped data
if (!ImportValidationUtilities.ValidateTemplateMapping(dt.Columns, mappingData))
{
// TODO: move this to front-end
response.Message = "Oops. Looks like you've uploaded the wrong template.";
response.StatusCode = 520;
response.ImportStatus.Messages.Add("<div>If you want to update your data, please use the 'Export' feature.</div>");
response.ImportStatus.Messages.Add(string.Format("<div>If you want to load new data, please use this <a href='{0}/Common/KPTemplates/2015_STeam_Goals_Template.xlsx'>Bulk Template</a> to bulk load your data.<br/>", this.spDataAccess.HostUrl));
response.ImportStatus.Status = "error";
return(response);
}
// validate that the data in the sheet template contains the
// key fields required for import and duplicate checking
// validate the mapping contains the required key fields (throws RequiredKeyFieldException)
// TODO: this should be part of the ValidateTemplateMapping
try
{
ImportValidationUtilities.ValidateRequiredKeyFields(mappings, this.primaryKeysList);
}
catch (RequiredKeyFieldException ex)
{
response.ImportStatus.Status = "error";
response.Errors.Add(new ErrorObject(ex.Message, ex.StackTrace));
response.Message = "Error: Required field exception";
response.StatusCode = 520;
return(response);
}
// clean-up data from spreadsheet and convert to a KPListItem object
// this will represent an entity object (e.g. Goal, Project, etc.)
List <KPListItem> sheetData = this.ConvertSheetData(dt, kpListColumns, mappingData, importStatus);
// checks for delimiters in values for Country & SecondaryVPs
ImportValidationUtilities.EnforceDelimiters(sheetData);
// validate the country and status values
ImportValidationUtilities.ValidateNonLookupFields(mappingData.ListName, sheetData, importStatus);
// copy original lookups for GoalSets
List <Dictionary <string, string> > originalLookups = new List <Dictionary <string, string> >();
// get the lookup values for the lookup fields
this.TransformLookupFieldValues(originalLookups, sheetData, importStatus);
// add each item
foreach (KPListItem sheetItem in sheetData)
{
// initialize flag to indicate if item was actually updated
bool isUpdated = false;
// get the team object from the worksheet for updating or creating a new item
Team userTeam = domainManager.User.Teams.Find(t => t.KPID == Int32.Parse(sheetItem["KPTeam"].Value));
// find the item if it already exists
int itemKPID = this.CheckForExistingItem(sheetItem, kpListItems, this.primaryKeysList);
// if we successfully found all matching Category L1 & L2 for the specified Goal Set then continue else we'll skip this row
if (ImportValidationUtilities.ValidateGoalSetCategories(this.domainManager, originalLookups[rowCount], sheetItem, importStatus))
{
// skip this if no team found
if (userTeam != null)
{
// is this a create or update?
if (itemKPID > -1)
{
// update the item and check flag for results
this.spDataAccess.UpdateEntityItem(userTeam.SiteUrl, mappingData.ListName, sheetItem, itemKPID, out isUpdated);
if (isUpdated)
{
importStatus.UpdatedRowCount++;
importStatus.Messages.Add(string.Format("Row {0}: Updated, new field values found.", rowCount + 1));
}
else
{
importStatus.SkippedRow(string.Format("Row {0}: Not updated, no changes found.", rowCount + 1));
}
}
else
{
this.spDataAccess.AddNewEntityItem(userTeam.SiteUrl, mappingData.ListName, sheetItem);
importStatus.CreatedRowCount++;
}
}
else
{
string errMsg = string.Format("Could not add item. Team '{0}' not found for current user.", userTeam.Nick);
importStatus.Messages.Add(errMsg);
importStatus.SkippedRow(errMsg);
}
}
else
{
importStatus.SkippedRowCount++;
}
rowCount++;
}
importStatus.TotalRowCount = rowCount;
response.Message = ImportConstants.SUCCESS_IMPORT;
response.ImportStatus = importStatus;
// catch the following errors
// skipped rows (what reason)?
// skipped columns (we only care about these if the values are mismatched - GoalSet/CL1/CL2/VP)
// Duplicate data (keys match existing data)
// message user
// rows created
// rows skipped - see above
// catch (UpdateOnlyTemplateException ex)
// {
// string ERR_ITEM_NOT_FOUND = "Error: Item not found to update (KPID:{0})";
// string err = string.Format(ERR_ITEM_NOT_FOUND, ex.Message);
// // add import exception errors for exceptions updating column values
// //importErrors.Add(new ErrorObject(err, string.Empty));
// //response.StatusCode = 206; // error
// }
// catch (ColumnUpdateException ex)
// {
// // add import exception errors for exceptions updating column values
// //importErrors.Add(new ErrorObject(ex.Message, ex.StackTrace));
// }
// catch (Exception ex)
// {
// // add validation errors
// //importErrors.Add(new ErrorObject(string.Format(ERR_ROW_MSG, rowIndex), ex.StackTrace));
// // error importing rows
// //response.StatusCode = 206; // error
// }
//}
//if (response.StatusCode < 400)
//{
// // TODO: refactor this - total mess
// switch (mappingData.ImportType)
// {
// case ImportType.Export:
// // successfully updated data from sheet
// if (rowIndex >= importCount)
// {
// response.Message = string.Format(ImportConstants.SUCCESS_IMPORT);
// response.Message += string.Format("Updated {0} of {1} rows from sheet.<br/>", importCount, rowIndex);
// }
// if (rowIndex > importCount)
// {
// // show skipped rows
// if (duplicateRows.Count > 0)
// response.Message += string.Format(ImportConstants.SUCCESS_SKIPPED_ROWS, duplicateRows.Count, string.Join(",", duplicateRows.ToArray()));
// }
// if (_versionConflict.Count > 0)
// {
// response.Message += string.Format("{0} Version Conflicts found:<br/>", _versionConflict.Count);
// response.Message += string.Join("<br/>", _versionConflict.ToArray());
// response.Message += "<br/>Please export the data and try again.<br/>";
// }
// if (_mismatchedLookups.Count > 0)
// {
// response.Message += string.Format("{0} Mismatched Lookup Values found:<br/>{1}", _mismatchedLookups.Count, string.Join("<br/>", _mismatchedLookups.ToArray()));
// response.StatusCode = 205; // warning
// }
// break;
// case ImportType.Dynamic:
// case ImportType.Template:
// // successfully imported data from sheet
// if (rowIndex >= importCount)
// {
// response.Message = string.Format(ImportConstants.SUCCESS_BULK_IMPORT);
// response.Message += string.Format("Created {0} of {1} rows from sheet.<br/>", importCount, rowIndex);
// }
// if (columnsMissing.Count > 0)
// {
// response.Message += "Bulk import successful ... but columns not found";
// response.Errors = columnsMissing;
// response.StatusCode = 205; // warning
// }
// if (_mismatchedLookups.Count > 0)
// {
// response.Message += string.Format("{0} Mismatched Lookup Values found:<br/>{1}", _mismatchedLookups.Count, string.Join("<br/>", _mismatchedLookups.ToArray()));
// response.StatusCode = 205; // warning
// }
// if (_incorrectLookups.Count > 0)
// {
// // show incorrect values mapped
// response.Message += string.Format("{0} Incorrect Lookup Values found:<br/>{1}", _incorrectLookups.Count, string.Join("<br/>", _incorrectLookups.ToArray()));
// response.Message += "<br/>Incorrect values were removed.";
// response.Message += "<br/>Please refer to Kingpin for the correct values to update.";
// response.StatusCode = 205; // warning
// }
// break;
// }
//}
//else
//{
// // an exception was thrown.
// response.Errors = importErrors;
//}
return(response);
}
private async Task <ImportResponse> Import(PublishDataProcessModel requestModel)
{
//var filePath = _importFilePath + "Market Size Import.xlsx";
var res = new ImportResponse();
List <ExcelUploadModel> objExcelUploadList = new List <ExcelUploadModel>();
//var inputModel = new MarketSizingImportInsertViewModel();
//IDbHelper _IDbHelper = new DbHelper(configuration);
//IImportRequestRepository iImportRequestRepository = new ImportRequestRepository(_IDbHelper);
//IImportRequestService iImportRequestService = new ImportRequestService(iImportRequestRepository);
try
{
//inputModel.TemplateName = _mSTemplateName;
//------------Call business method
// IMarketSizingImportExportRepository imarketSizingImportExportRepository = new MarketSizingImportExportRepository(_IDbHelper);
// IMarketSizingImportExportService imarketSizingImportExportService = new MarketSizingImportExportService(imarketSizingImportExportRepository);
// inputModel.UserCreatedById = Guid.Parse(_adminUserId);
var resultStag = false;
var rowCount = 0;
//------------Change Status to InProgress--------------------------
await UpdatePublishDataStatus(requestModel.Id, _appSettings.ImportStatusInProgress);
//ImportRequestViewModel objImportRequestViewModel = new ImportRequestViewModel();
//objImportRequestViewModel.Id = requestModel.Id;
//objImportRequestViewModel.ImportException = "Market sizing importe is processing.";
//objImportRequestViewModel.StatusName = _importStatusInprogress;
//iImportRequestService.Update(objImportRequestViewModel);
//--------------------------------------
//check file exist
using (var stream = File.OpenRead(requestModel.PublishFilePath))
{
//formFile.CopyTo(stream);
using (var package = new ExcelPackage(stream))
{
ExcelWorksheet worksheet = package.Workbook.Worksheets["Sheet1"];
if (worksheet.Dimension == null)
{
res.ResponseMessage = "Please upload proper excel file";
res.TotalSuccessRecords = 0;
res.TotalFailedRecords = 0;
res.ResponseStatus = "Failed";
return(res);
}
rowCount = worksheet.Dimension.Rows;
var rowFrom = 2;
for (int row = rowFrom; row <= rowCount; row++)
{
var objMarketSizingImport = ExcelCellToModel(worksheet, row);
objMarketSizingImport.WebsiteURL = requestModel.WebsiteURL;
objMarketSizingImport.UserName = requestModel.UserName;
objMarketSizingImport.Password = requestModel.Password;
objExcelUploadList.Add(objMarketSizingImport);
}
}
}
Parallel.ForEach(objExcelUploadList, item =>
{
//string postRes = PublishApiUtility.PostData(item);
//item.PublishResponse = postRes;
});
//}
res.TotalRecords = rowCount;
//var resError = result != null ? result.Where(s => s.ErrorNotes != null).ToList() : null;
//if (resError != null && resError.Count <= 0 || resError.First().ErrorNotes == null)
//{
// await UpdatePublishDataStatus(requestModel.Id, _appSettings.ImportStatusSuccess);
//}
//else
//{
// var errorFilePath = string.Empty;
// if (result != null && result.Count > 0)
// {
// errorFilePath = CreateMSExcel(result);
// }
// await UpdatePublishDataStatus(requestModel.Id, _appSettings.ImportStatusFailed);
//}
}
catch (Exception ex)
{
await UpdatePublishDataStatus(requestModel.Id, _appSettings.ImportStatusFailed);
//_logger.LogError(ex.Message);
//res.ResponseMessage = "Import() " + ex.Message;
//res.ResponseStatus = _importStatusFailed;
//var objImportRequestViewModel = new ImportRequestViewModel();
//objImportRequestViewModel.Id = requestModel.Id;
//objImportRequestViewModel.StatusName = _importStatusFailed;
//objImportRequestViewModel.ImportException = "Import marketSizing have some errors";
//objImportRequestViewModel.ImportExceptionFilePath = string.Empty;
//iImportRequestService.Update(objImportRequestViewModel);
}
return(res);
}
public ImportResponse ImportScheduledTaxRates(string fullFilePath)
{
var response = new ImportResponse();
if (string.IsNullOrWhiteSpace(fullFilePath))
{
response.AddError("No file specified");
return(response);
}
if (!fullFilePath.EndsWith(".csv"))
{
response.AddError("file must be in .csv format");
return(response);
}
if (!File.Exists(fullFilePath))
{
response.AddError("File does not exist");
return(response);
}
var lines = File.ReadAllLines(fullFilePath);
for (int i = 1; i < lines.Length; i++)
{
var lineParts = lines[i].Split(';');
if (lineParts.Length != 4)
{
response.AddFailedLine(i, "Not able to find all elements");
continue;
}
var municipality = lineParts[0];
var partsAreValid = true;
if (string.IsNullOrWhiteSpace(municipality))
{
response.AddFailedLine(i, "No municipality name specified");
partsAreValid = false;
}
if (!decimal.TryParse(lineParts[1], out decimal rate))
{
response.AddFailedLine(i, $"Unable to parse rate {lineParts[1]} to a decimal");
partsAreValid = false;
}
var frequencyParses = Enum.TryParse(lineParts[2], true, out Frequency frequency) && Enum.IsDefined(typeof(Frequency), frequency);
if (!frequencyParses)
{
response.AddFailedLine(i, $"Unable to parse {lineParts[2]} to a valid frequency");
partsAreValid = false;
}
if (frequencyParses)
{
var(Success, ErrorMessage, Ranges) = ParseDateRanges(lineParts[3], frequency);
if (!Success)
{
partsAreValid = false;
response.AddFailedLine(i, ErrorMessage);
}
if (!partsAreValid)
{
continue;
}
foreach (var(Start, End) in Ranges)
{
var scheduledTaxRate = _scheduledTaxRateFactory.Create(municipality, rate, frequency, Start, End);
_municipalityRepository.SaveScheduledTaxRate(scheduledTaxRate);
}
}
}
return(response);
}
/// <summary>
/// Validate the import mapping request
/// </summary>
/// <param name="spDataAccess"></param>
/// <param name="isImport"></param>
/// <returns></returns>
private MappingData ValidateRequest(SPDataAccess spDataAccess, bool isImport)
{
// get the message (body)
Message message = OperationContext.Current.RequestContext.RequestMessage;
ImportResponse response = new ImportResponse();
MappingData mappingData = null;
string jsonData = string.Empty;
// get the convert the body text
jsonData = JSON.GetPayload(message);
// deserialize the JSON body
mappingData = JsonConvert.DeserializeObject <MappingData>(jsonData);
if (string.IsNullOrEmpty(mappingData.ListName))
{
throw new Exception("List name is required (listName)");
}
if (string.IsNullOrEmpty(mappingData.FileName))
{
throw new Exception("File name is required (fileName)");
}
if (mappingData.TemplateType == null && isImport)
{
throw new Exception("Template type is required (templateType)");
}
if (mappingData.TemplateType.Equals(ImportType.Import) && mappingData.TeamId < 0)
{
throw new Exception("TeamId is required for Import templates");
}
// initialize user to get their teams
spDataAccess.InitializeCurrentUser();
// if we have a dynamic template then we need to set the
// team to what was passed in and add the required field to the mappings
if (mappingData.TemplateType.Equals(ImportType.Dynamic))
{
// get team
mappingData.Team = spDataAccess.CurrentUser.Teams.Find(t => t.KPID == mappingData.TeamId);
// create the mapping value with the delimiter
mappingData.Mappings.Add("KPTeam", string.Format("#{0}", mappingData.Team.Nick));
}
else
{
if (mappingData.TeamId.HasValue && mappingData.TeamId > -1)
{
Team team = spDataAccess.CurrentUser.Teams.Find(t => t.KPID == mappingData.TeamId);
if (team != null)
{
mappingData.Team = team;
}
else
{
throw new Exception(string.Format("Error Validating: TeamId {0} does not exist.", mappingData.TeamId));
}
}
else
{
if (spDataAccess.CurrentUser.Teams.Count > 0)
{
mappingData.Team = spDataAccess.CurrentUser.Teams[0];
}
else
{
throw new Exception(string.Format("Error Validating: User does not have any Teams assigned."));
}
}
}
return(mappingData);
}
public HttpResponseMessage MarkRecieptAsExported(string userName, string password, Guid receiptId)
{
var transactionResponse = new ImportResponse();
try
{
_log.InfoFormat("Login attempt for {0} - MarkReceiptAsExported", userName);
CostCentreLoginResponse response = _costCentreApplicationService.CostCentreLogin(userName, password, "HQAdmin");
AuditCCHit(response.CostCentreId, "Login", "Login attempt for ", response.ErrorInfo);
if (response.CostCentreId == Guid.Empty)
{
transactionResponse.Info = "Invalid user credentials";
transactionResponse.Status = false;
}
else
{
var data = _receiptExportDocumentRepository.MarkAsExported(receiptId);
if (data)
{
transactionResponse.Status = true;
}
else
{
transactionResponse.Info = "Failed to mark as exported";
transactionResponse.Status = false;
}
}
}
catch (Exception ex)
{
transactionResponse.Status = false;
transactionResponse.Info = "Error: An error occurred executing the task.Result details=>" + ex.Message + "Inner Exception:" + (ex.InnerException != null ? ex.InnerException.Message : "");
}
return Request.CreateResponse(HttpStatusCode.OK, transactionResponse);
}
public HttpResponseMessage GetNextReceiptToExport(string userName, string password)
{
var transactionResponse = new ImportResponse();
try
{
_log.InfoFormat("Login attempt for {0} - GetNextReceiptToExport", userName);
CostCentreLoginResponse response = _costCentreApplicationService.CostCentreLogin(userName, password, "HQAdmin");
AuditCCHit(response.CostCentreId, "Login", "Login attempt for ", response.ErrorInfo);
if (response.CostCentreId == Guid.Empty)
{
transactionResponse.Info = "Invalid user credentials";
transactionResponse.Status = false;
}
else
{
var data = _receiptExportDocumentRepository.GetPayment();
if (data != null)
{
transactionResponse.TransactionData = JsonConvert.SerializeObject(data);
transactionResponse.Status = true;
}
else
{
string documents = "Receipt" ;
transactionResponse.Info = string.Format("No {0} to import", documents);
transactionResponse.Status = false;
}
}
}
catch (Exception ex)
{
transactionResponse.Status = false;
transactionResponse.Info = "Error: An error occurred executing the task.Result details=>" + ex.Message + "Inner Exception:" + (ex.InnerException != null ? ex.InnerException.Message : "");
_log.Error(string.Format("Error: An error occurred when exporting transactions for {0}\n", ex));
}
return Request.CreateResponse(HttpStatusCode.OK, transactionResponse);
}
