/// <summary>
/// Generates the file schema for the csv parser.
/// </summary>
/// <returns>The Fileschema.</returns>
/// <remarks>Documented by Dev02, 2007-12-14</remarks>
private FileSchema GenerateFileSchema()
{
FileSchema fileSchema = new FileSchema();
fileSchema.Encoding = ((EncodingWrapper)comboBoxCharset.SelectedItem).Encoding;
fileSchema.hasHeaders = checkBoxHeader.Checked;
char delimiter;
switch (cmbFileFormat.SelectedIndex)
{
case 1:
delimiter = ';';
break;
case 2:
delimiter = '\t';
break;
case 3:
default:
delimiter = ',';
break;
}
fileSchema.Delimiter = delimiter;
return(fileSchema);
}
public string CreateFromTrackInfo(TrackInfo track)
{
string pattern = null;
try {
pattern = CreateFolderFilePattern(FolderSchema.Get(), FileSchema.Get());
} catch {}
return(CreateFromTrackInfo(pattern, track));
}
public LoadData CsvFile(
string filePath,
AnyValue key = null,
FileSyntax syntax = null,
FileSchema schema = null
)
{
SetConnectionOnClients();
return(Client.CsvFile(filePath, key, syntax, schema));
}
public LoadData CsvString(
string data,
AnyValue key = null,
FileSyntax syntax = null,
FileSchema schema = null
)
{
SetConnectionOnClients();
return(Client.JsonString(data, key, syntax, schema));
}
public bool LoadCSV(
string rel,
string data = null,
string path = null,
AnyValue[] key = null,
FileSyntax syntax = null,
FileSchema schema = null,
Integration integration = null
)
{
SetConnectionOnClients();
return(Client.LoadCSV(rel, data, path, key, syntax, schema, integration));
}
public bool LoadEdb(
string rel,
string contentType,
string data = null,
string path = null,
AnyValue key = null,
FileSyntax syntax = null,
FileSchema schema = null
)
{
SetConnectionOnClients();
return(Client.LoadEdb(rel, contentType, data, path, key, syntax, schema));
}
public static void Run([EventGridTrigger] EventGridEvent eventGridEvent, ILogger log)
{
System.Text.StringBuilder ErrorRecords = new System.Text.StringBuilder();
System.Text.StringBuilder AcceptedRecords = new System.Text.StringBuilder();
RootObject rootObject = Newtonsoft.Json.JsonConvert.DeserializeObject <RootObject>(eventGridEvent.Data.ToString());
if (rootObject.url.Contains("/LandBlob/"))
{
string blobName = rootObject.url.Substring(rootObject.url.IndexOf("/claimshark/")).Replace("/claimshark/", "");
string PayerID = blobName.Split("/")[0];
if (!CheckFileExists(PayerID, System.IO.Path.GetFileName(blobName)))
{
CloudStorageAccount mycloudStorageAccount = CloudStorageAccount.Parse(Environment.GetEnvironmentVariable("StorageConnection"));
CloudBlobClient blobClient = mycloudStorageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference("claimshark");
CloudBlob cloudBlockBlob = container.GetBlobReference(blobName);
string text;
List <FileError> fileErrors = new List <FileError>();
using (var stream = new System.IO.MemoryStream())
{
cloudBlockBlob.DownloadToStreamAsync(stream).Wait();
text = System.Text.Encoding.UTF8.GetString(stream.ToArray());
}
if (!String.IsNullOrEmpty(text))
{
var binDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
string PayerSchemaFile = Path.Combine(binDirectory, PayerID + ".json");
string PayerSchema;
if (System.IO.File.Exists(PayerSchemaFile))
{
PayerSchema = File.ReadAllText(PayerSchemaFile);
}
else
{
binDirectory = binDirectory.Substring(0, binDirectory.LastIndexOf(@"\"));
PayerSchemaFile = Path.Combine(binDirectory, PayerID + ".json");
PayerSchema = File.ReadAllText(PayerSchemaFile);
}
FileSchema fileSchema = Newtonsoft.Json.JsonConvert.DeserializeObject <FileSchema>(PayerSchema);
CreateTable(fileSchema.Fileds);
string[] data = text.Split('\n');
int LineNo = 1;
foreach (var value in data)
{
if (value.Trim().Length > 0)
{
System.Data.DataRow dataRow = dtPayerDataTable.NewRow();
dataRow["IsValid"] = 1;
if (fileSchema.FieldSeprator == "FixedLength")
{
int startPosition = 1;
int fieldLength = 0;
foreach (var field in fileSchema.Fileds)
{
fieldLength = Convert.ToInt32(field.Length);
if (value.Length >= (startPosition + fieldLength))
{
dataRow[field.FieldName] = value.Substring(startPosition - 1, fieldLength);
if (field.IsRequired == "Yes")
{
if (value.Substring(startPosition - 1, fieldLength).Trim().Length <= 0)
{
fileErrors.Add(new FileError()
{
FieldName = field.FieldName,
StartingPostiton = startPosition.ToString(),
FileLinePosition = LineNo.ToString()
});
dataRow["IsValid"] = 0;
}
}
}
else
{
if (value.Length > startPosition)
{
dataRow[field.FieldName] = value.Substring(startPosition - 1);
if (field.IsRequired == "Yes")
{
if (value.Substring(startPosition - 1).Trim().Length <= 0)
{
fileErrors.Add(new FileError()
{
FieldName = field.FieldName,
StartingPostiton = startPosition.ToString(),
FileLinePosition = LineNo.ToString()
});
dataRow["IsValid"] = 0;
}
}
break;
}
}
startPosition += fieldLength;
}
dtPayerDataTable.Rows.Add(dataRow);
}
}
if (fileErrors.Count > 0)
{
ErrorRecords.Append(value);
}
else
{
AcceptedRecords.Append(value);
}
LineNo += 1;
}
}
LogFileDetails(fileErrors, blobName, PayerID);
UploadBlob(fileErrors, ErrorRecords, AcceptedRecords, blobName, PayerID);
SendEmail(blobName, PayerID);
log.LogInformation(Newtonsoft.Json.JsonConvert.SerializeObject(fileErrors));
}
}
}
public void CanParseBytesFromStream()
{
var schema = new FileSchema();
var expected = new byte[] {0x0d, 0x0a};
var field = new Field(schema, new MemoryStream(expected));
var actualStream = (Stream)field.Value;
var actual = new byte[actualStream.Length];
actualStream.Read(actual, 0, actual.Length);
Assert.IsTrue(actual.SequenceEqual(expected));
}
public void CanParseBytesFromBase64String()
{
var input = @"Cg0W/w==";
var expected = new byte[] {0x0a, 0x0d, 0x16, 0xff};
var schema = new FileSchema
{
Default = input,
};
var field = new Field(schema);
var actualStream = (Stream)field.Value;
var actual = new byte[actualStream.Length];
actualStream.Read(actual, 0, actual.Length);
Assert.IsTrue(actual.SequenceEqual(expected));
}
