public ReaderFlatFile(Connection connection, FlatFile table, bool previewMode)
{
ReferenceConnection = connection;
_fileConnection = (ConnectionFlatFile)connection;
CacheTable = table;
_previewMode = previewMode;
switch (table.FormatType)
{
case DataType.ETypeCode.Json:
_fileHandler = new FileHandlerJson(table, table.RowPath);
break;
case DataType.ETypeCode.Text:
_fileHandler = new FileHandlerText(table, table.FileConfiguration);
break;
case DataType.ETypeCode.Xml:
_fileHandler = new FileHandlerXml(table, table.RowPath);
break;
default:
throw new ConnectionException(
$"The format type ${table.FormatType} is not currently supported.");
}
_fileNameOrdinal = table.GetDeltaColumnOrdinal(TableColumn.EDeltaType.FileName);
_baseRow = new object[table.Columns.Count];
}
private void OnProcessingStarted(FileHandlerBase inputHandler)
{
textBlockPreview.Text = inputHandler.GetPreview();
if (SWITCH_TO_LOG_TAB_ON_PROCESSING_START.Value)
{
m_tabItemOnStarted.IsSelected = true;
}
UpdateButtonStates();
}
private static void OnProcessingStarted(FileHandlerBase inputHandler)
{
c_isProcessing = true;
var handler = ProcessingStarted;
if (handler != null)
{
handler(inputHandler);
}
}
public ProcessingSet(FileHandlerBase inputFile)
{
m_id = IdentityManager.AcquireIdentity(GetType().Name);
m_inputHandler = inputFile;
var tiledPath = PointCloudTileSource.GetTileSourcePath(m_inputHandler.FilePath);
m_tiledHandler = LASFile.Create(tiledPath, null);
Directory.CreateDirectory(Path.GetDirectoryName(m_tiledHandler.FilePath));
}
private static void StartNextInProcessingQueue()
{
if (c_queue.Count > 0)
{
if (!c_backgroundWorker.IsBusy)
{
FileHandlerBase inputHandler = c_queue.Dequeue();
OnProcessingStarted(inputHandler);
c_backgroundWorker.RunWorkerAsync(inputHandler);
}
}
}
public void FileNameMustBeUsedAsSourceId()
{
//Arrange
var newId = Guid.NewGuid();
var fileName = string.Format("C:\\test\\{0}.xlsx", newId);
//Act
var actualValue = FileHandlerBase <DataEntryDataContract> .GetSourceId(fileName);
//Assert
Assert.AreEqual(newId, actualValue);
}
private static void Water(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj)
{
var data = new WaterDataContract
{
CostCode = obj.CostCode,
EndDate = obj.EndDate,
Money = obj.Money,
RowNo = obj.RowNo,
SourceId = obj.SourceId,
StartDate = obj.StartDate,
Units = obj.Units
};
handler.CallService <IWaterService>(service => service.UpsertDataEntry(data));
}
private static void Electricity(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj)
{
var data = new ElectricityDataContract
{
CostCode = obj.CostCode,
EndDate = obj.EndDate,
Money = obj.Money,
RowNo = obj.RowNo,
SourceId = obj.SourceId,
StartDate = obj.StartDate,
Units = obj.Units,
ElectricityType = ElectricityType.SouthAfricanNationalGrid
};
handler.CallService <IElectricityService>(service => service.UpsertDataEntry(data));
}
private static void AirTravelBusiness(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj)
{
var data = new AirTravelDataContract
{
CostCode = obj.CostCode,
EndDate = obj.EndDate,
Money = obj.Money,
RowNo = obj.RowNo,
SourceId = obj.SourceId,
StartDate = obj.StartDate,
Units = obj.Units,
TravelClass = TravelClass.Business
};
handler.CallService <IAirTravelService>(service => service.UpsertDataEntry(data));
}
private static void CommutingTrain(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj)
{
var data = new CommutingDataContract
{
CostCode = obj.CostCode,
EndDate = obj.EndDate,
Money = obj.Money,
RowNo = obj.RowNo,
SourceId = obj.SourceId,
StartDate = obj.StartDate,
Units = obj.Units,
CommutingType = CommutingType.Train
};
handler.CallService <ICommutingService>(service => service.UpsertDataEntry(data));
}
private static void CarHireP(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj)
{
var data = new CarHireDataContract
{
CostCode = obj.CostCode,
EndDate = obj.EndDate,
Money = obj.Money,
RowNo = obj.RowNo,
SourceId = obj.SourceId,
StartDate = obj.StartDate,
Units = obj.Units,
CarGroupBill = CarGroupBill.P
};
handler.CallService <ICarHireService>(service => service.UpsertDataEntry(data));
}
private static void Refrigerant410A(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj)
{
var data = new RefrigerantDataContract
{
CostCode = obj.CostCode,
EndDate = obj.EndDate,
Money = obj.Money,
RowNo = obj.RowNo,
SourceId = obj.SourceId,
StartDate = obj.StartDate,
Units = obj.Units,
RefrigerantType = RefrigerantType.R410A
};
handler.CallService <IRefrigerantService>(service => service.UpsertDataEntry(data));
}
private static void FuelCoalDomesticTonnes(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj)
{
var data = new FuelDataContract
{
CostCode = obj.CostCode,
EndDate = obj.EndDate,
Money = obj.Money,
RowNo = obj.RowNo,
SourceId = obj.SourceId,
StartDate = obj.StartDate,
Units = obj.Units,
FuelType = WCF.Fuel.FuelType.CoalDomestic,
UOM = UnitOfMeasure.Tonnes
};
handler.CallService <IFuelService>(service => service.UpsertDataEntry(data));
}
private static void PaperMondiA3Tonnes(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj)
{
var data = new PaperDataContract
{
CostCode = obj.CostCode,
EndDate = obj.EndDate,
Money = obj.Money,
RowNo = obj.RowNo,
SourceId = obj.SourceId,
StartDate = obj.StartDate,
Units = obj.Units,
PaperType = PaperType.MondiA3,
PaperUom = PaperUom.Tonnes
};
handler.CallService <IPaperService>(service => service.UpsertDataEntry(data));
}
private static void FleetPetrolThirdParty(FileHandlerBase <GenericDataContract> handler, GenericDataContract obj)
{
var data = new FleetDataContract
{
CostCode = obj.CostCode,
EndDate = obj.EndDate,
Money = obj.Money,
RowNo = obj.RowNo,
SourceId = obj.SourceId,
StartDate = obj.StartDate,
Units = obj.Units,
FuelType = FuelType.Petrol,
Scope = FleetScope.ThirdParty
};
handler.CallService <IFleetService>(service => service.UpsertDataEntry(data));
}
/*
* public static void Write(String output, String fileName) {
* Handler textHandler = getHandler(fileName);
* if (textHandler != null) textHandler.Write(output);
* }
*/
private bool CanWriteStringToFile(String output, String fileName)
{
if (!Ready)
{
// This would never log if filemanager not ready
//Log.Error("Filemanager not initialized", "CanWriteStringToFile");
return(false);
}
if (String.IsNullOrWhiteSpace(fileName))
{
Log.Error("Null or empty filename", "CanWriteStringToFile");
return(false);
}
// Note: SE silently fails when filename has invalid chars
int firstIllegalChar = fileName.IndexOfAny(Path.GetInvalidFileNameChars());
if (firstIllegalChar >= 0)
{
Log.Error("Illegal filename character at position " + firstIllegalChar, "CanWriteStringToFile");
return(false);
}
if (String.IsNullOrWhiteSpace(output))
{
Log.Warning("Null or empty output", "CanWriteStringToFile");
return(false);
}
FileHandlerBase handler = getHandler(fileName);
if (handler == null)
{
Log.Error("Error retrieving handler", "CanWriteStringToFile");
return(false);
}
//Log.Trace("Can write.", "CanWriteStringToFile");
//Log.Trace("ouput: " + output, "CanWriteStringToFile");
return(true);
}
public void Read <T>(String fileName, ref T result)
{
//Log.Trace(String.Format("Read file \"{0}\"", fileName), "Read");
if (!Exists(fileName))
{
Log.Trace("No file exists for " + fileName, "Read");
return;
}
FileHandlerBase handler = getHandler(fileName);
if (handler == null)
{
Log.Trace("Unable to acquire handler for " + fileName, "Read");
return;
}
handler.Read <T>(ref result);
}
public static string GetBinarySourceName(FileHandlerBase handler)
{
return(string.Format("{0}.{1}", handler.FilePath, PointCloudBinarySource.FILE_EXTENSION));
}
private static void AddTileSource(PointCloudTileSource tileSource, FileHandlerBase inputHandler)
{
tileSource.Open();
c_sources.Add(tileSource, inputHandler);
}
/// <summary>
/// This performns a lookup directly against the underlying data source, returns the result, and adds the result to cache.
/// </summary>
/// <param name="table"></param>
/// <param name="filters"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <ICollection <object[]> > LookupRow(Table table, List <Filter> filters, CancellationToken cancellationToken)
{
try
{
var restFunction = (WebService)table;
var baseRow = new object[table.Columns.Count];
var response = await GetWebServiceResponse(restFunction, filters, cancellationToken);
var responseStatusOrdinal = restFunction.GetDeltaColumnOrdinal(TableColumn.EDeltaType.ResponseStatus);
var responseSuccessOrdinal = restFunction.GetDeltaColumnOrdinal(TableColumn.EDeltaType.ResponseSuccess);
var responseDataOrdinal = restFunction.GetDeltaColumnOrdinal(TableColumn.EDeltaType.ResponseData);
var urlOrdinal = restFunction.GetDeltaColumnOrdinal(TableColumn.EDeltaType.Url);
var errorOrdinal = restFunction.GetDeltaColumnOrdinal(TableColumn.EDeltaType.Error);
var lookupResult = new List <object[]>();
if (responseStatusOrdinal >= 0)
{
baseRow[responseStatusOrdinal] = response.statusCode;
}
if (responseSuccessOrdinal >= 0)
{
baseRow[responseSuccessOrdinal] = response.isSuccess;
}
if (urlOrdinal >= 0)
{
baseRow[urlOrdinal] = response.url;
}
foreach (var column in restFunction.Columns.Where(c => c.IsInput))
{
if (filters != null)
{
var filter = filters.Where(c => c.Column1.Name == column.Name).ToArray();
if (!filter.Any())
{
baseRow[restFunction.GetOrdinal(column)] = column.DefaultValue;
}
else
{
baseRow[restFunction.GetOrdinal(column)] = filter.First().Value2;
}
}
else
{
baseRow[restFunction.GetOrdinal(column)] = column.DefaultValue;
}
}
if (response.isSuccess)
{
FileHandlerBase fileHanlder = null;
if (restFunction.FormatType == ETypeCode.Json)
{
fileHanlder = new FileHandlerJson(restFunction, restFunction.RowPath);
}
if (restFunction.FormatType == ETypeCode.Xml)
{
fileHanlder = new FileHandlerXml(restFunction, restFunction.RowPath);
}
if (fileHanlder != null)
{
await fileHanlder.SetStream(response.response, null);
return(await fileHanlder.GetAllRows(baseRow));
}
}
else
{
if (errorOrdinal >= 0)
{
var reader = new StreamReader(response.response);
var errorString = await reader.ReadToEndAsync();
baseRow[errorOrdinal] = errorString;
}
return(new[] { baseRow });
}
throw new ConnectionException($"The lookup failed as the web service format type {restFunction.FormatType} is not currently supported.");
}
catch (Exception ex)
{
throw new ConnectionException($"Lookup on the web service {table.Name} failed. {ex.Message}", ex);
}
}
public override async Task <bool> Open(long auditKey, SelectQuery query, CancellationToken cancellationToken)
{
AuditKey = auditKey;
try
{
if (_isOpen)
{
throw new ConnectionException("The information hub connection is already open.");
}
var downloadUrl = await _dexihConnection.GetDownloadUrl();
var intanceId = await _dexihConnection.GetRemoteAgentInstanceId();
// call the central web server to requet the query start.
var message = Json.SerializeObject(new
{
HubName = ReferenceConnection.DefaultDatabase,
CacheTable.SourceConnectionName,
TableName = CacheTable.Name,
TableSchema = CacheTable.Schema,
Query = query,
DownloadUrl = downloadUrl,
InstanceId = intanceId
}, "");
var content = new StringContent(message, Encoding.UTF8, "application/json");
var response = await _dexihConnection.HttpPost("OpenTableQuery", content);
if ((bool)response["success"])
{
_dataUrl = response["value"].ToString();
}
else
{
throw new ConnectionException($"Error {response?["message"]}", new Exception(response["exceptionDetails"].ToString()));
}
// use the returned url, to start streaming the data.
using (var httpClient = new HttpClient())
{
var response2 = await httpClient.GetAsync(_dataUrl, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
if (response2.StatusCode == HttpStatusCode.InternalServerError)
{
var responseString = await response2.Content.ReadAsStringAsync();
var result = JObject.Parse(responseString);
var returnValue = result.ToObject <ReturnValue>();
throw new ConnectionException("Dexih Reader Failed. " + returnValue.Message,
returnValue.Exception);
}
var responseStream = await response2.Content.ReadAsStreamAsync();
var config = new FileConfiguration();
_fileHandler = new FileHandlerText(CacheTable, config);
await _fileHandler.SetStream(responseStream, null);
_baseRow = new object[CacheTable.Columns.Count];
return(true);
}
}
catch (Exception ex)
{
throw new ConnectionException($"Opening connection to information hub failed. {ex.Message}", ex);
}
}
public override async Task <Table> GetSourceTableInfo(Table originalTable, CancellationToken cancellationToken)
{
try
{
var flatFile = (FlatFile)originalTable;
if (flatFile.FileConfiguration == null || flatFile.FileSample == null)
{
throw new ConnectionException($"The properties have not been set to import the flat files structure. Required properties are (FileFormat)FileFormat and (Stream)FileSample.");
}
var stream = new MemoryStream();
var writer = new StreamWriter(stream);
writer.Write(flatFile.FileSample);
writer.Flush();
stream.Position = 0;
FileHandlerBase fileHandler = null;
switch (flatFile.FormatType)
{
case ETypeCode.Json:
fileHandler = new FileHandlerJson(flatFile, flatFile.RowPath);
break;
case ETypeCode.Text:
fileHandler = new FileHandlerText(flatFile, flatFile.FileConfiguration);
break;
case ETypeCode.Xml:
fileHandler = new FileHandlerXml(flatFile, flatFile.RowPath);
break;
default:
throw new ConnectionException($"The source type {flatFile.FormatType} is not currently supported.");
}
var columns = await fileHandler.GetSourceColumns(stream);
//The new datatable that will contain the table schema
var newFlatFile = new FlatFile();
flatFile.Name = originalTable.Name;
newFlatFile.Columns.Clear();
newFlatFile.LogicalName = newFlatFile.Name;
newFlatFile.Description = "";
newFlatFile.FileConfiguration = flatFile.FileConfiguration;
foreach (var column in columns)
{
newFlatFile.Columns.Add(column);
}
var col = new TableColumn()
{
//add the basic properties
Name = "FileName",
LogicalName = "FileName",
IsInput = false,
DataType = ETypeCode.String,
DeltaType = TableColumn.EDeltaType.FileName,
Description = "The name of the file the record was loaded from.",
AllowDbNull = false,
IsUnique = false
};
newFlatFile.Columns.Add(col);
return(newFlatFile);
}
catch (Exception ex)
{
throw new ConnectionException($"Failed to import the file structure. {ex.Message}", ex);
}
}
