public ReaderFlatFile(Connection connection, FlatFile table, bool previewMode) { ReferenceConnection = connection; _fileConnection = (ConnectionFlatFile)connection; CacheTable = table; _previewMode = previewMode; switch (table.FormatType) { case DataType.ETypeCode.Json: _fileHandler = new FileHandlerJson(table, table.RowPath); break; case DataType.ETypeCode.Text: _fileHandler = new FileHandlerText(table, table.FileConfiguration); break; case DataType.ETypeCode.Xml: _fileHandler = new FileHandlerXml(table, table.RowPath); break; default: throw new ConnectionException( $"The format type ${table.FormatType} is not currently supported."); } _fileNameOrdinal = table.GetDeltaColumnOrdinal(TableColumn.EDeltaType.FileName); _baseRow = new object[table.Columns.Count]; }
private void OnProcessingStarted(FileHandlerBase inputHandler) { textBlockPreview.Text = inputHandler.GetPreview(); if (SWITCH_TO_LOG_TAB_ON_PROCESSING_START.Value) { m_tabItemOnStarted.IsSelected = true; } UpdateButtonStates(); }
private static void OnProcessingStarted(FileHandlerBase inputHandler) { c_isProcessing = true; var handler = ProcessingStarted; if (handler != null) { handler(inputHandler); } }
public ProcessingSet(FileHandlerBase inputFile) { m_id = IdentityManager.AcquireIdentity(GetType().Name); m_inputHandler = inputFile; var tiledPath = PointCloudTileSource.GetTileSourcePath(m_inputHandler.FilePath); m_tiledHandler = LASFile.Create(tiledPath, null); Directory.CreateDirectory(Path.GetDirectoryName(m_tiledHandler.FilePath)); }
private static void StartNextInProcessingQueue() { if (c_queue.Count > 0) { if (!c_backgroundWorker.IsBusy) { FileHandlerBase inputHandler = c_queue.Dequeue(); OnProcessingStarted(inputHandler); c_backgroundWorker.RunWorkerAsync(inputHandler); } } }
public void FileNameMustBeUsedAsSourceId() { //Arrange var newId = Guid.NewGuid(); var fileName = string.Format("C:\\test\\{0}.xlsx", newId); //Act var actualValue = FileHandlerBase <DataEntryDataContract> .GetSourceId(fileName); //Assert Assert.AreEqual(newId, actualValue); }
private static void Water(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj) { var data = new WaterDataContract { CostCode = obj.CostCode, EndDate = obj.EndDate, Money = obj.Money, RowNo = obj.RowNo, SourceId = obj.SourceId, StartDate = obj.StartDate, Units = obj.Units }; handler.CallService <IWaterService>(service => service.UpsertDataEntry(data)); }
private static void Electricity(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj) { var data = new ElectricityDataContract { CostCode = obj.CostCode, EndDate = obj.EndDate, Money = obj.Money, RowNo = obj.RowNo, SourceId = obj.SourceId, StartDate = obj.StartDate, Units = obj.Units, ElectricityType = ElectricityType.SouthAfricanNationalGrid }; handler.CallService <IElectricityService>(service => service.UpsertDataEntry(data)); }
private static void AirTravelBusiness(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj) { var data = new AirTravelDataContract { CostCode = obj.CostCode, EndDate = obj.EndDate, Money = obj.Money, RowNo = obj.RowNo, SourceId = obj.SourceId, StartDate = obj.StartDate, Units = obj.Units, TravelClass = TravelClass.Business }; handler.CallService <IAirTravelService>(service => service.UpsertDataEntry(data)); }
private static void CommutingTrain(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj) { var data = new CommutingDataContract { CostCode = obj.CostCode, EndDate = obj.EndDate, Money = obj.Money, RowNo = obj.RowNo, SourceId = obj.SourceId, StartDate = obj.StartDate, Units = obj.Units, CommutingType = CommutingType.Train }; handler.CallService <ICommutingService>(service => service.UpsertDataEntry(data)); }
private static void CarHireP(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj) { var data = new CarHireDataContract { CostCode = obj.CostCode, EndDate = obj.EndDate, Money = obj.Money, RowNo = obj.RowNo, SourceId = obj.SourceId, StartDate = obj.StartDate, Units = obj.Units, CarGroupBill = CarGroupBill.P }; handler.CallService <ICarHireService>(service => service.UpsertDataEntry(data)); }
private static void Refrigerant410A(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj) { var data = new RefrigerantDataContract { CostCode = obj.CostCode, EndDate = obj.EndDate, Money = obj.Money, RowNo = obj.RowNo, SourceId = obj.SourceId, StartDate = obj.StartDate, Units = obj.Units, RefrigerantType = RefrigerantType.R410A }; handler.CallService <IRefrigerantService>(service => service.UpsertDataEntry(data)); }
private static void FuelCoalDomesticTonnes(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj) { var data = new FuelDataContract { CostCode = obj.CostCode, EndDate = obj.EndDate, Money = obj.Money, RowNo = obj.RowNo, SourceId = obj.SourceId, StartDate = obj.StartDate, Units = obj.Units, FuelType = WCF.Fuel.FuelType.CoalDomestic, UOM = UnitOfMeasure.Tonnes }; handler.CallService <IFuelService>(service => service.UpsertDataEntry(data)); }
private static void PaperMondiA3Tonnes(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj) { var data = new PaperDataContract { CostCode = obj.CostCode, EndDate = obj.EndDate, Money = obj.Money, RowNo = obj.RowNo, SourceId = obj.SourceId, StartDate = obj.StartDate, Units = obj.Units, PaperType = PaperType.MondiA3, PaperUom = PaperUom.Tonnes }; handler.CallService <IPaperService>(service => service.UpsertDataEntry(data)); }
private static void FleetPetrolThirdParty(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj) { var data = new FleetDataContract { CostCode = obj.CostCode, EndDate = obj.EndDate, Money = obj.Money, RowNo = obj.RowNo, SourceId = obj.SourceId, StartDate = obj.StartDate, Units = obj.Units, FuelType = FuelType.Petrol, Scope = FleetScope.ThirdParty }; handler.CallService <IFleetService>(service => service.UpsertDataEntry(data)); }
/* * public static void Write(String output, String fileName) { * Handler textHandler = getHandler(fileName); * if (textHandler != null) textHandler.Write(output); * } */ private bool CanWriteStringToFile(String output, String fileName) { if (!Ready) { // This would never log if filemanager not ready //Log.Error("Filemanager not initialized", "CanWriteStringToFile"); return(false); } if (String.IsNullOrWhiteSpace(fileName)) { Log.Error("Null or empty filename", "CanWriteStringToFile"); return(false); } // Note: SE silently fails when filename has invalid chars int firstIllegalChar = fileName.IndexOfAny(Path.GetInvalidFileNameChars()); if (firstIllegalChar >= 0) { Log.Error("Illegal filename character at position " + firstIllegalChar, "CanWriteStringToFile"); return(false); } if (String.IsNullOrWhiteSpace(output)) { Log.Warning("Null or empty output", "CanWriteStringToFile"); return(false); } FileHandlerBase handler = getHandler(fileName); if (handler == null) { Log.Error("Error retrieving handler", "CanWriteStringToFile"); return(false); } //Log.Trace("Can write.", "CanWriteStringToFile"); //Log.Trace("ouput: " + output, "CanWriteStringToFile"); return(true); }
public void Read <T>(String fileName, ref T result) { //Log.Trace(String.Format("Read file \"{0}\"", fileName), "Read"); if (!Exists(fileName)) { Log.Trace("No file exists for " + fileName, "Read"); return; } FileHandlerBase handler = getHandler(fileName); if (handler == null) { Log.Trace("Unable to acquire handler for " + fileName, "Read"); return; } handler.Read <T>(ref result); }
public static string GetBinarySourceName(FileHandlerBase handler) { return(string.Format("{0}.{1}", handler.FilePath, PointCloudBinarySource.FILE_EXTENSION)); }
private static void AddTileSource(PointCloudTileSource tileSource, FileHandlerBase inputHandler) { tileSource.Open(); c_sources.Add(tileSource, inputHandler); }
/// <summary> /// This performns a lookup directly against the underlying data source, returns the result, and adds the result to cache. /// </summary> /// <param name="table"></param> /// <param name="filters"></param> /// <param name="cancellationToken"></param> /// <returns></returns> public async Task <ICollection <object[]> > LookupRow(Table table, List <Filter> filters, CancellationToken cancellationToken) { try { var restFunction = (WebService)table; var baseRow = new object[table.Columns.Count]; var response = await GetWebServiceResponse(restFunction, filters, cancellationToken); var responseStatusOrdinal = restFunction.GetDeltaColumnOrdinal(TableColumn.EDeltaType.ResponseStatus); var responseSuccessOrdinal = restFunction.GetDeltaColumnOrdinal(TableColumn.EDeltaType.ResponseSuccess); var responseDataOrdinal = restFunction.GetDeltaColumnOrdinal(TableColumn.EDeltaType.ResponseData); var urlOrdinal = restFunction.GetDeltaColumnOrdinal(TableColumn.EDeltaType.Url); var errorOrdinal = restFunction.GetDeltaColumnOrdinal(TableColumn.EDeltaType.Error); var lookupResult = new List <object[]>(); if (responseStatusOrdinal >= 0) { baseRow[responseStatusOrdinal] = response.statusCode; } if (responseSuccessOrdinal >= 0) { baseRow[responseSuccessOrdinal] = response.isSuccess; } if (urlOrdinal >= 0) { baseRow[urlOrdinal] = response.url; } foreach (var column in restFunction.Columns.Where(c => c.IsInput)) { if (filters != null) { var filter = filters.Where(c => c.Column1.Name == column.Name).ToArray(); if (!filter.Any()) { baseRow[restFunction.GetOrdinal(column)] = column.DefaultValue; } else { baseRow[restFunction.GetOrdinal(column)] = filter.First().Value2; } } else { baseRow[restFunction.GetOrdinal(column)] = column.DefaultValue; } } if (response.isSuccess) { FileHandlerBase fileHanlder = null; if (restFunction.FormatType == ETypeCode.Json) { fileHanlder = new FileHandlerJson(restFunction, restFunction.RowPath); } if (restFunction.FormatType == ETypeCode.Xml) { fileHanlder = new FileHandlerXml(restFunction, restFunction.RowPath); } if (fileHanlder != null) { await fileHanlder.SetStream(response.response, null); return(await fileHanlder.GetAllRows(baseRow)); } } else { if (errorOrdinal >= 0) { var reader = new StreamReader(response.response); var errorString = await reader.ReadToEndAsync(); baseRow[errorOrdinal] = errorString; } return(new[] { baseRow }); } throw new ConnectionException($"The lookup failed as the web service format type {restFunction.FormatType} is not currently supported."); } catch (Exception ex) { throw new ConnectionException($"Lookup on the web service {table.Name} failed. {ex.Message}", ex); } }
public override async Task <bool> Open(long auditKey, SelectQuery query, CancellationToken cancellationToken) { AuditKey = auditKey; try { if (_isOpen) { throw new ConnectionException("The information hub connection is already open."); } var downloadUrl = await _dexihConnection.GetDownloadUrl(); var intanceId = await _dexihConnection.GetRemoteAgentInstanceId(); // call the central web server to requet the query start. var message = Json.SerializeObject(new { HubName = ReferenceConnection.DefaultDatabase, CacheTable.SourceConnectionName, TableName = CacheTable.Name, TableSchema = CacheTable.Schema, Query = query, DownloadUrl = downloadUrl, InstanceId = intanceId }, ""); var content = new StringContent(message, Encoding.UTF8, "application/json"); var response = await _dexihConnection.HttpPost("OpenTableQuery", content); if ((bool)response["success"]) { _dataUrl = response["value"].ToString(); } else { throw new ConnectionException($"Error {response?["message"]}", new Exception(response["exceptionDetails"].ToString())); } // use the returned url, to start streaming the data. using (var httpClient = new HttpClient()) { var response2 = await httpClient.GetAsync(_dataUrl, HttpCompletionOption.ResponseHeadersRead, cancellationToken); if (response2.StatusCode == HttpStatusCode.InternalServerError) { var responseString = await response2.Content.ReadAsStringAsync(); var result = JObject.Parse(responseString); var returnValue = result.ToObject <ReturnValue>(); throw new ConnectionException("Dexih Reader Failed. " + returnValue.Message, returnValue.Exception); } var responseStream = await response2.Content.ReadAsStreamAsync(); var config = new FileConfiguration(); _fileHandler = new FileHandlerText(CacheTable, config); await _fileHandler.SetStream(responseStream, null); _baseRow = new object[CacheTable.Columns.Count]; return(true); } } catch (Exception ex) { throw new ConnectionException($"Opening connection to information hub failed. {ex.Message}", ex); } }
public override async Task <Table> GetSourceTableInfo(Table originalTable, CancellationToken cancellationToken) { try { var flatFile = (FlatFile)originalTable; if (flatFile.FileConfiguration == null || flatFile.FileSample == null) { throw new ConnectionException($"The properties have not been set to import the flat files structure. Required properties are (FileFormat)FileFormat and (Stream)FileSample."); } var stream = new MemoryStream(); var writer = new StreamWriter(stream); writer.Write(flatFile.FileSample); writer.Flush(); stream.Position = 0; FileHandlerBase fileHandler = null; switch (flatFile.FormatType) { case ETypeCode.Json: fileHandler = new FileHandlerJson(flatFile, flatFile.RowPath); break; case ETypeCode.Text: fileHandler = new FileHandlerText(flatFile, flatFile.FileConfiguration); break; case ETypeCode.Xml: fileHandler = new FileHandlerXml(flatFile, flatFile.RowPath); break; default: throw new ConnectionException($"The source type {flatFile.FormatType} is not currently supported."); } var columns = await fileHandler.GetSourceColumns(stream); //The new datatable that will contain the table schema var newFlatFile = new FlatFile(); flatFile.Name = originalTable.Name; newFlatFile.Columns.Clear(); newFlatFile.LogicalName = newFlatFile.Name; newFlatFile.Description = ""; newFlatFile.FileConfiguration = flatFile.FileConfiguration; foreach (var column in columns) { newFlatFile.Columns.Add(column); } var col = new TableColumn() { //add the basic properties Name = "FileName", LogicalName = "FileName", IsInput = false, DataType = ETypeCode.String, DeltaType = TableColumn.EDeltaType.FileName, Description = "The name of the file the record was loaded from.", AllowDbNull = false, IsUnique = false }; newFlatFile.Columns.Add(col); return(newFlatFile); } catch (Exception ex) { throw new ConnectionException($"Failed to import the file structure. {ex.Message}", ex); } }