} // When no entity identifier is provided
public static StreamFormat CreateFromFile(string path)
{
if (string.IsNullOrEmpty(path))
{
throw (new ArgumentNullException(nameof(path)));
}
string configPath = path.Contains(Path.DirectorySeparatorChar) ? path : $"{AppDomain.CurrentDomain.BaseDirectory}{path}";
var jsonString = File.ReadAllText(configPath);
try
{
var streamObj = JsonConvert.DeserializeObject <NarrowBatchFormat>(jsonString);
// Test that must have entityKey or entityIdentifier
if (String.IsNullOrEmpty(streamObj.entityIdentifier) && String.IsNullOrEmpty(streamObj.entityKey))
{
throw (new ArgumentException("Must specify an entityIdentifier column or an entityKey"));
}
dynamic newObj = null;
// Not narrow
if (String.IsNullOrEmpty(streamObj.signalIdentifier))
{
// Not batch
if (String.IsNullOrEmpty(streamObj.batchIdentifier))
{
// No entityKey
if (String.IsNullOrEmpty(streamObj.entityKey))
{
newObj = new StreamFormat
{
entityIdentifier = streamObj.entityIdentifier,
timeIdentifier = streamObj.timeIdentifier,
timeFormat = streamObj.timeFormat,
timeZone = streamObj.timeZone
};
}
else
{
newObj = new StreamFormat
{
entityKey = streamObj.entityKey,
timeIdentifier = streamObj.timeIdentifier,
timeFormat = streamObj.timeFormat,
timeZone = streamObj.timeZone
};
}
}
else
{
// No entityKey
if (String.IsNullOrEmpty(streamObj.entityKey))
{
newObj = new BatchFormat
{
batchIdentifier = streamObj.batchIdentifier,
entityIdentifier = streamObj.entityIdentifier,
timeIdentifier = streamObj.timeIdentifier,
timeFormat = streamObj.timeFormat,
timeZone = streamObj.timeZone
};
}
else
{
newObj = new BatchFormat
{
batchIdentifier = streamObj.batchIdentifier,
entityKey = streamObj.entityKey,
timeIdentifier = streamObj.timeIdentifier,
timeFormat = streamObj.timeFormat,
timeZone = streamObj.timeZone
};
}
}
}
else
{
// Not batch
if (String.IsNullOrEmpty(streamObj.batchIdentifier))
{
if (String.IsNullOrEmpty(streamObj.entityKey))
{
newObj = new NarrowFormat
{
entityIdentifier = streamObj.entityIdentifier,
timeIdentifier = streamObj.timeIdentifier,
timeFormat = streamObj.timeFormat,
timeZone = streamObj.timeZone,
signalIdentifier = streamObj.signalIdentifier,
valueIdentifier = streamObj.valueIdentifier
};
}
else
{
newObj = new NarrowFormat
{
entityKey = streamObj.entityKey,
timeIdentifier = streamObj.timeIdentifier,
timeFormat = streamObj.timeFormat,
timeZone = streamObj.timeZone,
signalIdentifier = streamObj.signalIdentifier,
valueIdentifier = streamObj.valueIdentifier
};
}
}
}
return(newObj ?? streamObj);
}
catch (Exception e)
{
throw (e);
}
}
static void Main(string[] args)
{
// Create logger
var logDir = $"{AppDomain.CurrentDomain.BaseDirectory}Log";
Log.Logger = new LoggerConfiguration()
.MinimumLevel.Debug()
.WriteTo.Console(restrictedToMinimumLevel: LogEventLevel.Information)
.WriteTo.File(logDir + $"{Path.DirectorySeparatorChar}logfile.log", rollingInterval: RollingInterval.Day)
.CreateLogger();
// Arg Parsing
string rootPath = null;
string configPath = null;
string streamName = null;
string datastream = null;
string api = null;
string account = null;
string token = null;
uint? batchSize = DEFAULTBATCH;
uint? sleep = DEFAULTSLEEP;
string fileFilter = "*.csv";
uint? chunkMB = DEFAULCHUNK;
Parser.Default.ParseArguments <Options>(args).WithParsed <Options>(o =>
{
rootPath = o.RootPath;
configPath = o.ConfigPath;
streamName = o.StreamName;
datastream = o.StreamId;
api = o.Uri;
token = o.Token;
account = o.Account;
fileFilter = o.FilesFilter ?? fileFilter;
sleep = o.Sleep ?? sleep;
batchSize = o.Batch ?? batchSize;
chunkMB = o.ChunkMB ?? chunkMB;
}).WithNotParsed <Options>((errs) => Environment.Exit(-1));
// Clamp sleep and batch
if (sleep < 0)
{
sleep = DEFAULTSLEEP;
}
if (batchSize <= 0)
{
batchSize = DEFAULTBATCH;
}
// Validate that name of id specified
if (String.IsNullOrEmpty(streamName) && String.IsNullOrEmpty(datastream))
{
throw (new ArgumentException("Must specify a stream name or a stream id."));
}
// This will be loaded from defined config file
Log.Information($"Reading config file ");
var sf = StreamFormat.CreateFromFile(configPath);
var timeZone = sf.timeZone;
var isBatchModel = (sf is BatchFormat || sf is NarrowBatchFormat);
Log.Information("Starting up CSV Loader");
string jsonText = "";
try
{
var acctProps = new Dictionary <string, string>();
// acctProps["tercel.connector.flush.signal.batch.size"] = "25";
var apiHelper = new ApiHelper(account, token, api, acctProps, responseCallback:
(response) => {
jsonText = JsonConvert.SerializeObject(response, Formatting.Indented,
new JsonSerializerSettings
{
ReferenceLoopHandling = ReferenceLoopHandling.Ignore
//ReferenceLoopHandling=ReferenceLoopHandling.Serialize,
//PreserveReferencesHandling=PreserveReferencesHandling.Objects
});
if (jsonText.Contains("excep"))
{
Log.Error(jsonText);
}
else
{
Log.Debug(jsonText);
}
});
// var exist = apiHelper.GetAccountProps(account);
// Create datastream only if not exits
if (String.IsNullOrEmpty(datastream))
{
dynamic stream = apiHelper.SetDataStream($"{streamName}", timeZone, isBatchModel ? MODELTYPE.BATCH : MODELTYPE.SLIDING_WINDOW);
datastream = (string)stream.id;
}
// Get all csv files
var info = new DirectoryInfo(rootPath);
var filesInfo = info.GetFiles(fileFilter, SearchOption.AllDirectories);
//var files = from file in Directory.EnumerateFiles(rootPath, "*cleansed.csv", SearchOption.AllDirectories) select file;
var files = from file in filesInfo.Where(f => f.Length > 0) select file.FullName;
// files = from file in files where file.Contains("SignalsOutput") && !file.Contains("batches") select file;
// var files = from file in Directory.EnumerateFiles(rootPath, "*entity.csv", SearchOption.AllDirectories) select file;
Console.WriteLine($"{files.Count().ToString()} files found.");
var spec = new IngestSpec <StreamFormat>()
{
format = sf
};
var job = new Job <StreamFormat>()
{
jobType = JOBTYPE.INGESTDATA,
status = JOBSTATUS.CREATED,
datastream = datastream,
spec = spec
};
try
{
var loadResponse = apiHelper.LoadCSVFiles(files.ToList <string>(), job, (int)batchSize, (int)sleep, chunkMB.Value);
jsonText = JsonConvert.SerializeObject(loadResponse, Formatting.Indented,
new JsonSerializerSettings
{
ReferenceLoopHandling = ReferenceLoopHandling.Ignore
//ReferenceLoopHandling=ReferenceLoopHandling.Serialize,
//PreserveReferencesHandling=PreserveReferencesHandling.Objects
});
}
catch (Exception e)
{
// Print outer exception
printExceptionInformation(e);
}
}
catch (Exception ex)
{
Console.WriteLine($"Exception reading csv files from directory: {rootPath}");
printExceptionInformation(ex);
}
Log.Debug("Shutting down CSV Loader");
var dir = $"{AppDomain.CurrentDomain.BaseDirectory}Responses";
if (!Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
File.WriteAllText($"{dir}{Path.DirectorySeparatorChar}responses.txt", jsonText);
return;
}
