/* public QueryReport Insert_DATA() { QueryReport result = new QueryReport(); result.StartReport(); Dictionary<string, List<string>> _view = null; // return view of csv file if (_fileCsvData != null && _fileCsvData.IsValid) { result.FileSource = _fileCsvData.filename; string[] filesData = System.IO.Directory.GetFiles(_fileCsvData.filePath); using (System.IO.StreamReader streamFile = new System.IO.StreamReader(filesData[0])) { _view = Mapping.GetView(streamFile.BaseStream, _mapping, 0, (_mapping.CSV_HASHEADER)?1:0); streamFile.Close(); } QueryReport _result = ISTAT.DBDAL.DataAccess.Insert_Data(this.DataSet.IDSet, _view); result.Merge(_result); } // return view of xml file if (_filesXmlData != null) { result.FileSource = _filesXmlData.filename; string[] filesData = System.IO.Directory.GetFiles(_filesXmlData.filePath); using (System.IO.StreamReader streamFile = new System.IO.StreamReader(filesData[0])) { TranscodeTime transcode = (_mapping != null) ? (_mapping.TranscodeUse) ? _mapping.TranscodeTime : null : null; _view = ISTAT.DBDAL.DataSDMX.GetView(streamFile.BaseStream, transcode, 0); streamFile.Close(); } QueryReport _result = ISTAT.DBDAL.DataAccess.Insert_Data(this.DataSet.IDSet, _view); result.Merge(_result); } // Write log on db for RSS if (result.RecordCount>0 || result.RecordOverrideCount>0) ISTAT.DBDAL.DataAccess.Insert_Load( this.DataSet.IDSet, this.DataSet.IDFlow, result.MinTime, result.MaxTime, result.RecordCount+result.RecordOverrideCount, result.FileSource); result.StopReport(); return result; }*/ public QueryReport Insert_DATA() { QueryReport result = new QueryReport(); result.StartReport(); Dictionary<string, List<string>> _view = null; try { // return view of csv file if (_fileCsvData != null && _fileCsvData.IsValid) { result.FileSource = _fileCsvData.filename; QueryReport _result = ISTAT.DBDAL.DataAccess.Insert_CSVData(_fileCsvData.filePath + @"\" + _fileCsvData.filename, _fileCsvData.separator, _mapping.IDSchema); result.Merge(_result); } // return view of xml file if (_filesXmlData != null) { result.FileSource = _filesXmlData.filename; string[] filesData = System.IO.Directory.GetFiles(_filesXmlData.filePath); using (System.IO.StreamReader streamFile = new System.IO.StreamReader(filesData[0])) { string tempCsvFilename = Path.GetTempFileName(); TranscodeTime transcode = (_mapping != null) ? (_mapping.TranscodeUse) ? _mapping.TranscodeTime : null : null; System.IO.StreamWriter outFile = new System.IO.StreamWriter(tempCsvFilename); var fieldsList = ISTAT.DBDAL.DataSDMX.WriteSDMXtoCSV(streamFile.BaseStream, outFile); //var builder = SDMX_Dataloader.Engine.BuilderProcedure.Create(); //var mapping = builder.Create_MappingForSdmxMl(_ds.IDSet, fieldsList); QueryReport _result = ISTAT.DBDAL.DataAccess.Insert_CSVData(tempCsvFilename, ';', 0, _ds.IDSet); //builder.Delete_MAPPING(mapping.IDSchema); if (outFile.BaseStream != null) outFile.Close(); System.IO.File.Delete(tempCsvFilename); result.Merge(_result); } } } catch (Exception ex) { QueryReport _report = new QueryReport(); _report.RecordCount = 0; _report.RecordTargetCount = 0; _report.Errors.Add(ex.Message); result.Merge(_report); } // Write log on db for RSS if (result.RecordCount > 0 || result.RecordOverrideCount > 0) ISTAT.DBDAL.DataAccess.Insert_Load( this.DataSet.IDSet, this.DataSet.IDFlow, result.MinTime, result.MaxTime, result.RecordCount + result.RecordOverrideCount, result.FileSource); result.StopReport(); return result; }
public QueryReport Insert_AttributeDATA(string PathDir) { QueryReport result = new QueryReport(); result.StartReport(); if (_fileCsvData != null && _fileCsvData.IsValid) { result.FileSource = _fileCsvData.filename; QueryReport _result = ISTAT.DBDAL.DataAccess.Insert_AttributeData(_fileCsvData.filePath + @"\" + _fileCsvData.filename, this.DataSet.IDSet, PathDir); result.Merge(_result); } if (result.RecordCount > 0 || result.RecordOverrideCount > 0) ISTAT.DBDAL.DataAccess.Insert_Load( this.DataSet.IDSet, this.DataSet.IDFlow, result.MinTime, result.MaxTime, result.RecordCount + result.RecordOverrideCount, result.FileSource); result.StopReport(); return result; }
public QueryReport Insert_DATA() { QueryReport result = new QueryReport(); result.StartReport(); Dictionary<string, List<string>> _view = null; // return view of csv file if (_fileCsvData != null && _fileCsvData.IsValid) { result.FileSource = _fileCsvData.filename; string[] filesData = System.IO.Directory.GetFiles(_fileCsvData.filePath); using (System.IO.StreamReader streamFile = new System.IO.StreamReader(filesData[0])) { _view = Mapping.GetView(streamFile.BaseStream, _mapping, 0, (_mapping.CSV_HASHEADER)?1:0); streamFile.Close(); } QueryReport _result = ISTAT.DBDAL.DataAccess.Insert_Data(this.DataSet.IDSet, _view); result.Merge(_result); } // return view of xml file if (_filesXmlData != null) { result.FileSource = _filesXmlData.filename; string[] filesData = System.IO.Directory.GetFiles(_filesXmlData.filePath); using (System.IO.StreamReader streamFile = new System.IO.StreamReader(filesData[0])) { TranscodeTime transcode = (_mapping != null) ? (_mapping.TranscodeUse) ? _mapping.TranscodeTime : null : null; _view = ISTAT.DBDAL.DataSDMX.GetView(streamFile.BaseStream, transcode, 0); streamFile.Close(); } QueryReport _result = ISTAT.DBDAL.DataAccess.Insert_Data(this.DataSet.IDSet, _view); result.Merge(_result); } // Write log on db for RSS if (result.RecordCount>0 || result.RecordOverrideCount>0) ISTAT.DBDAL.DataAccess.Insert_Load( this.DataSet.IDSet, this.DataSet.IDFlow, result.MinTime, result.MaxTime, result.RecordCount+result.RecordOverrideCount, result.FileSource); result.StopReport(); return result; }
/// <summary> /// Insert_Data /// </summary> /// <param name="idset">ID of Structure </param> /// <param name="view">Matrix like view of CUBE</param> /// <returns></returns> /// public static QueryReport Insert_Data(int idset, Dictionary<string, List<string>> view) { QueryReport _report = new QueryReport(); _report.StartReport(); string min_time = string.Empty; string max_time = string.Empty; try { #region INIT VARIABLE QueryPart fi_dim_qp = new QueryPart(); QueryPart fi_att_qp = new QueryPart(); QueryPart fa_att_qp = new QueryPart(); List<ConceptDimensionDB> lstDim = DataAccess.Get_DimensionsPartial(idset); List<ConceptAttributeDB> lstAtt = DataAccess.Get_AttributesPartial(idset); int SID = -1; int itemCount = ((List<string>)view[view.Keys.First<string>()]).Count; _report.RecordTargetCount = itemCount; #endregion #region LOAD TABLE TIME PERIOD // Chache of TIME_PERIOD db table List<TimePeriod> time_period = DataAccess.Get_TimePeriod(); #endregion for (int i = 0; i < itemCount; i++) { bool hasError = false; QueryPart cur_fi_dim_qp = new QueryPart(); QueryPart cur_fi_att_qp = new QueryPart(); QueryPart cur_fa_att_qp = new QueryPart(); #region GET DIMENSIONS PART foreach (ConceptDimensionDB conc in lstDim) { // se la dimensione non e' una timedimension if (!conc.IsTimeSeriesDim) { cur_fi_dim_qp[conc.Code] = new QueryCouple(conc.MemberTable, view[conc.Code][i]); } } #endregion #region GET ATTRIBUTES PART foreach (ConceptAttributeDB conc in lstAtt) { if (view.ContainsKey(conc.Code)) { if (conc.IsObservationValue) { cur_fa_att_qp[conc.Code] = new QueryCouple(conc.MemberTable, view[conc.Code][i]); } else { cur_fi_att_qp[conc.Code] = new QueryCouple(conc.MemberTable, view[conc.Code][i]); } } } #endregion if (!hasError) { #region GET FILTS SID KEY if (cur_fi_dim_qp.KEY_RECORD != fi_dim_qp.KEY_RECORD) { // Select SID from view SID = DataAccess.Get_SIDToFilts(idset, cur_fi_dim_qp); _report.QueryExecCount++; // if no Serie found if (SID < 0) { #region RESOLVE FILTS KEY foreach (string dimStr in cur_fi_dim_qp.FIELDS.Keys) { // risolvo il codice della dimensione ed ottengo l'id nel database cur_fi_dim_qp[dimStr].VALUE = DataAccess.Get_IDCode(cur_fi_dim_qp[dimStr].MEMBER, cur_fi_dim_qp[dimStr].CODE); _report.QueryExecCount++; if ((int)cur_fi_dim_qp[dimStr].VALUE < 0) { // codice non rivolvibile _report.Errors.Add(string.Format("Unable to resolve {0} in row [{1}]", dimStr, i)); hasError = true; break; } fi_dim_qp = cur_fi_dim_qp; } #endregion if (!hasError) { #region Attribute FILTS Query Part if (cur_fi_att_qp.KEY_RECORD != fi_att_qp.KEY_RECORD) { foreach (string attStr in cur_fi_att_qp.FIELDS.Keys) { cur_fi_att_qp[attStr].VALUE = DataAccess.Get_IDCode(cur_fi_att_qp[attStr].MEMBER, cur_fi_att_qp[attStr].CODE); _report.QueryExecCount++; if ((int)cur_fi_att_qp[attStr].VALUE < 0) { var isCodeRappresentation = (from c in lstAtt where c.MemberTable == cur_fi_att_qp[attStr].MEMBER select c.IsCodelist).OfType<bool>().First(); if (!isCodeRappresentation) cur_fi_att_qp[attStr].VALUE = DataAccess.Insert_Code("Att", cur_fi_att_qp[attStr].MEMBER, cur_fi_att_qp[attStr].CODE, string.Empty); if ((int)cur_fi_att_qp[attStr].VALUE < 0) cur_fi_att_qp[attStr].VALUE = null; /* if ((int)cur_fi_att_qp[attStr].VALUE < 0) { // codice non rivolvibile hasError = true; _report.Errors.Add(string.Format("Unable to resolve {0} in row [{1}]", attStr, i)); break; } */ } } fi_att_qp = cur_fi_att_qp; } #endregion //Insert new series in FILTS SID = DataAccess.Insert_DataFilts(idset, fi_dim_qp.SELECT_PART + ((fi_att_qp.SELECT_PART != string.Empty) ? " , " + fi_att_qp.SELECT_PART : string.Empty), fi_dim_qp.INSERT_PART_VALUE + ((fi_att_qp.INSERT_PART_VALUE != string.Empty) ? "," + fi_att_qp.INSERT_PART_VALUE : string.Empty), fi_dim_qp.WHERE_PART_VALUE); _report.QueryExecCount++; } else { // Serie non trovata } } fi_dim_qp = cur_fi_dim_qp; } #endregion if (!hasError) { #region Attribute FACTS Query Part if (cur_fa_att_qp.KEY_RECORD != fa_att_qp.KEY_RECORD) { foreach (string attStr in cur_fa_att_qp.FIELDS.Keys) { cur_fa_att_qp[attStr].VALUE = DataAccess.Get_IDCode(cur_fa_att_qp[attStr].MEMBER, cur_fa_att_qp[attStr].CODE); _report.QueryExecCount++; if ((int)cur_fa_att_qp[attStr].VALUE < 0) { var isCodeRappresentation = (from c in lstAtt where c.MemberTable == cur_fa_att_qp[attStr].MEMBER select c.IsCodelist).OfType<bool>().First(); if (!isCodeRappresentation) cur_fa_att_qp[attStr].VALUE = DataAccess.Insert_Code("Att", cur_fa_att_qp[attStr].MEMBER, cur_fa_att_qp[attStr].CODE, string.Empty); if ((int)cur_fa_att_qp[attStr].VALUE < 0) cur_fa_att_qp[attStr].VALUE = null; // codice non rivolvibile /* if ((int)cur_fa_att_qp[attStr].VALUE < 0) { _report.Errors.Add(string.Format("Unable to resolve {0} in row [{1}]", attStr, i)); hasError = true; break; } */ } } fa_att_qp = cur_fa_att_qp; } #endregion } if (!hasError) { if (SID > -1) { #region INSERT INTO FactS int idtime = -1; string tPeriod = TranscodeTimePeriod(view["TIME_PERIOD"][i], view["FREQ"][i]); try { idtime = (from period in time_period where period.CODE == tPeriod select period.ID).First(); } catch { idtime = -1; } if (idtime >= 0) { // Converto value per inserimento in db Sql object value = (view["OBS_VALUE"][i] == string.Empty || view["OBS_VALUE"][i] == "NaN") ? null : view["OBS_VALUE"][i].Replace('.', ','); //DataAccess.Delete_DataFacts(idset, SID, idtime); int _res = DataAccess.Insert_DataFacts(idset, SID, idtime, value, fa_att_qp.SELECT_PART, fa_att_qp.INSERT_PART_VALUE); _report.QueryExecCount++; // Get Min - Max Time min_time = TranscodeTime.CompareMin(min_time, tPeriod); max_time = TranscodeTime.CompareMax(max_time, tPeriod); switch (_res) { case 0: _report.RecordCount++; break; case 2: _report.RecordOverrideCount++; break; case 1: _report.RecordIgnoreCount++; break; case -1: _report.Errors.Add("Insert fail on Record [" + i + "]"); break; } } else { _report.Errors.Add(string.Format("Unable to resolve TIME_PERIOD : {0} in row [{1}]", tPeriod, i)); hasError = true; } #endregion } } } } } catch (Exception ex) { Console.Write(ex.Message); } finally { _report.StopReport(); _report.MaxTime = max_time; _report.MinTime = min_time; } return _report; }
/* private static int Insert_ImplicitMapping(int idset, List<string> sdmxmlFields) { HashSet<string> cubeConcepts = new HashSet<string>(); Dictionary<string, Dictionary<string, object>> dict = new Dictionary<string, Dictionary<string, object>>(); ISTAT.DBDAL.DataAccess.Get_Dimensions(idset).ForEach(e => { cubeConcepts.Add(e.Id); }); ISTAT.DBDAL.DataAccess.Get_Attributes(idset).ForEach(e => { cubeConcepts.Add(e.Id); }); cubeConcepts.Add("OBS_VALUE"); if (!cubeConcepts.Contains("TIME_PERIOD")) cubeConcepts.Add("TIME_PERIOD"); int pos = 1; foreach (string field in sdmxmlFields) { if (cubeConcepts.Contains(field)) { //MappingItem i = new MappingItem(); //i._a = pos; //i._b = field; //i._c = "Cln: " + pos + " : " + field; //items.Add(i); Dictionary<string, object> c = new Dictionary<string, object>(); c.Add(field, "Cln: " + pos + " : " + field); dict.Add(pos.ToString(), c); } pos++; } int idSchema = -1; if ((idSchema = ISTAT.DBDAL.DataAccess.Create_Mapping( idset, 0, "ImplicitSdmxMlMapping", "Implicit SDMX-ML Mapping", dict, ";", true, false, "", 0)) > 0) { //OK } return idSchema; }*/ public static QueryReport Insert_CSVData(string csvFilePath, char csvDelimiter, int mappingID, int idset = -1) { var configDebug = ConfigurationManager.AppSettings["DebugBulkUpload"]; bool debug = (configDebug == "true") ? true : false; var bulkuploadTimeout = ConfigurationManager.AppSettings["BulkUploadTimeout"]; int timeout = 0; Int32.TryParse(bulkuploadTimeout, out timeout); QueryReport _report = new QueryReport(); _report.StartReport(); string min_time = string.Empty; string max_time = string.Empty; string temp_table = "temp_upload_" + Utility.RandomString(16); int rowcount = 0; int proc_result = 0; DataWrapper dtw = new DataWrapper(DataWrapper.ECONNECTIONTYPE.SQL, DataAccess.SQLConnString_DB.ConnectionString); if (!dtw.TestConnection()) return null; CSVUploader uploader; if (debug) { uploader = new CSVUploader((System.Data.SqlClient.SqlConnection)dtw.DBConnection); uploader.CreateDestinationTable = true; uploader.ConnectionTimeOut = timeout; uploader.CSVDelimiter = (mappingID > 0) ? csvDelimiter : ';'; //Se il mappingID non è definito (implicit mapping) utilizzo sempre ';' uploader.Upload(csvFilePath, temp_table); dtw.DBConnection.Close(); } dtw.DBConnection.Open(); IDbTransaction transaction = dtw.DBConnection.BeginTransaction(); try { if (mappingID <= 0 && idset <= 0) { throw new Exception("Ivalid call of Insert_CSVData with mappingID=" + mappingID + " and idset=" + idset + ". Either a valid 'mappingID' or a valid 'idset' must be provided"); } // Carico il file csv nel DB if (!debug) { uploader = new CSVUploader((System.Data.SqlClient.SqlConnection)dtw.DBConnection, (System.Data.SqlClient.SqlTransaction)transaction); uploader.CreateDestinationTable = true; uploader.ConnectionTimeOut = timeout; uploader.CSVDelimiter = (mappingID > 0) ? csvDelimiter : ';'; //Se il mappingID non è definito (implicit mapping) utilizzo sempre ';' uploader.Upload(csvFilePath, temp_table); } // Lancio la SP per l'upload System.Data.IDbCommand cmd = dtw.DBConnection.CreateCommand(); cmd.Connection = dtw.DBConnection; cmd.Transaction = transaction; cmd.CommandType = System.Data.CommandType.StoredProcedure; cmd.CommandText = "proc_bulkupload_UPLOAD_DATA"; System.Data.IDbDataParameter _param_code = cmd.CreateParameter(); _param_code.DbType = System.Data.DbType.Int64; _param_code.ParameterName = "@IDStructure"; _param_code.Value = (mappingID > 0) ? mappingID : idset; //Se il mappingID non è definito (implicit mapping) utilizzo l'idset cmd.Parameters.Add(_param_code); System.Data.IDbDataParameter _param_table_name = cmd.CreateParameter(); _param_table_name.DbType = System.Data.DbType.String; _param_table_name.ParameterName = "@table_name"; _param_table_name.Value = temp_table; cmd.Parameters.Add(_param_table_name); System.Data.IDbDataParameter _param_rowcount = cmd.CreateParameter(); _param_rowcount.DbType = System.Data.DbType.Int32; _param_rowcount.Direction = ParameterDirection.Output; _param_rowcount.ParameterName = "@rowcount"; cmd.Parameters.Add(_param_rowcount); System.Data.IDbDataParameter _param_drop = cmd.CreateParameter(); _param_drop.DbType = System.Data.DbType.Int32; _param_drop.ParameterName = "@dropTable"; _param_drop.Value = (debug) ? 0 : 1; cmd.Parameters.Add(_param_drop); System.Data.IDbDataParameter _param_implicit_map = cmd.CreateParameter(); _param_implicit_map.DbType = System.Data.DbType.Int32; _param_implicit_map.ParameterName = "@UseImplicitMapping"; _param_implicit_map.Value = (mappingID > 0) ? 0 : 1; cmd.Parameters.Add(_param_implicit_map); System.Data.IDataParameter _return_val = cmd.CreateParameter(); _return_val.DbType = System.Data.DbType.Int32; _return_val.ParameterName = "@errorCode"; _return_val.Direction = ParameterDirection.ReturnValue; cmd.Parameters.Add(_return_val); //proc_result = cmd.ExecuteNonQuery(); cmd.CommandTimeout = timeout; System.Data.IDataReader dataReader = cmd.ExecuteReader(); //Check for errors int errors = 0; while (dataReader.Read()) { string csv_col = dataReader.GetString(dataReader.GetOrdinal("CSV_COLUMN")); string code = dataReader.GetString(dataReader.GetOrdinal("CODE")); string error = dataReader.GetString(dataReader.GetOrdinal("ERROR")); errors++; _report.Errors.Add("CONCEPT: " + csv_col + ", CODE: " + code + ", ERROR: " + error); } dataReader.Close(); rowcount = (Int32)_param_rowcount.Value; proc_result = (Int32)_return_val.Value; if (errors == 0 && proc_result == 0) { transaction.Commit(); _report.RecordCount = rowcount; _report.RecordTargetCount = rowcount; } else { transaction.Rollback(); _report.Errors.Add("Errore " + proc_result); _report.RecordCount = 0; _report.RecordTargetCount = rowcount; } } catch (Exception ex) { _report.Errors.Add(ex.Message); try { transaction.Rollback(); } catch { } } finally { _report.StopReport(); _report.MaxTime = max_time; _report.MinTime = min_time; dtw.DBConnection.Close(); } return _report; }
public static QueryReport Insert_AttributeData(string attributeFilePath, int idSet, string userPathDir) { QueryReport _report = new QueryReport(); _report.StartReport(); string userScartedFileDir = userPathDir + @"\scarted"; string userScartedFileName = attributeFilePath.Substring(attributeFilePath.LastIndexOf(@"\") + 1).Replace(".csv", "_scarted.csv"); string min_time = string.Empty; string max_time = string.Empty; DataWrapper dtw = new DataWrapper(DataWrapper.ECONNECTIONTYPE.SQL, DataAccess.SQLConnString_DB.ConnectionString); if (!dtw.TestConnection()) return null; dtw.DBConnection.Open(); try { // Carico il file csv nel DB CSVUploader uploader = new CSVUploader(SQLConnString_DB.ConnectionString); uploader.CreateDestinationTable = true; uploader.ConnectionTimeOut = 3600; uploader.Upload(attributeFilePath, "Temp_Attribute"); // Lancio la SP per l'update degli attributi System.Data.IDbCommand cmd = dtw.DBConnection.CreateCommand(); cmd.CommandType = System.Data.CommandType.StoredProcedure; cmd.CommandText = "sp_AttributeMain"; System.Data.IDbDataParameter _param_code = cmd.CreateParameter(); _param_code.DbType = System.Data.DbType.String; _param_code.ParameterName = "@IDSET"; _param_code.Value = idSet; cmd.Parameters.Add(_param_code); cmd.ExecuteNonQuery(); // Faccio una select sulla tabella degli scarti // se il risultato non è null con i dati creo un file csv e restituisco il link cmd = dtw.DBConnection.CreateCommand(); cmd.CommandText = "SELECT * FROM Temp_Attribute_Scarted"; System.Data.IDataReader dReader = cmd.ExecuteReader(); string sHeader = "", sRow = ""; bool bHeader = true; if (!Directory.Exists(userPathDir)) Directory.CreateDirectory(userPathDir); if (!Directory.Exists(userScartedFileDir)) Directory.CreateDirectory(userScartedFileDir); File.Create(userScartedFileDir + @"\" + userScartedFileName).Close(); while (dReader.Read()) { sRow = ""; using (System.IO.StreamWriter file = new System.IO.StreamWriter(userScartedFileDir + @"\" + userScartedFileName, true)) { if (bHeader) { for (int i = 0; i < dReader.FieldCount; i++) { sHeader += dReader.GetName(i) + "|"; } bHeader = false; sHeader = sHeader.Substring(0, sHeader.Length - 1); file.WriteLine(sHeader); _report.ScartedFilePath = @"/scarted/" + userScartedFileName; } for (int i = 0; i < dReader.FieldCount; i++) { sRow += dReader[i].ToString() + "|"; } sRow = sRow.Substring(0, sRow.Length - 1); file.WriteLine(sRow); } } } catch (Exception ex) { _report.Errors.Add(ex.Message); } finally { _report.StopReport(); _report.MaxTime = max_time; _report.MinTime = min_time; } return _report; }