static List <ParseWorker> StartParseWorkers(BuildConf conf)
{
var parse_workers = new List <ParseWorker>();
int files_per_worker = conf.files.Count < conf.max_threads ? conf.files.Count : (int)Math.Ceiling((float)conf.files.Count / (float)conf.max_threads);
int idx = 0;
int wid = 0;
while (idx < conf.files.Count)
{
int count = (idx + files_per_worker) > conf.files.Count ? (conf.files.Count - idx) : files_per_worker;
++wid;
var pw = new ParseWorker();
pw.id = wid;
pw.self_file = conf.self_file;
pw.start = idx;
pw.inc_path.Add(conf.inc_dir);
pw.count = count;
pw.use_cache = conf.use_cache;
pw.cache_dir = conf.tmp_dir;
pw.check_deps = conf.check_deps;
pw.files = conf.files;
parse_workers.Add(pw);
pw.Start();
idx += count;
}
return(parse_workers);
}
static bool CheckArgsSignatureFile(BuildConf conf)
{
var tmp_args_file = conf.tmp_dir + "/" + Path.GetFileName(conf.res_file) + ".args";
bool changed = !File.Exists(tmp_args_file) || (File.Exists(tmp_args_file) && File.ReadAllText(tmp_args_file) != conf.args);
if (changed)
{
File.WriteAllText(tmp_args_file, conf.args);
}
return(changed);
}
int DoExec(BuildConf conf)
{
var res_dir = Path.GetDirectoryName(conf.res_file);
if (res_dir.Length > 0)
{
Directory.CreateDirectory(res_dir);
}
Directory.CreateDirectory(conf.tmp_dir);
var args_changed = CheckArgsSignatureFile(conf);
if (conf.use_cache &&
!args_changed &&
!BuildUtil.NeedToRegen(conf.res_file, conf.files)
)
{
Console.WriteLine("BHL no need to re-build");
return(0);
}
var ts = conf.ts;
if (ts == null)
{
ts = new Types();
}
conf.userbindings.Register(ts);
var parse_workers = StartParseWorkers(conf);
var compiler_workers = StartAndWaitCompileWorkers(conf, ts, parse_workers);
var tmp_res_file = conf.tmp_dir + "/" + Path.GetFileName(conf.res_file) + ".tmp";
if (!WriteCompilationResultToFile(conf, compiler_workers, tmp_res_file))
{
return(ERROR_EXIT_CODE);
}
if (File.Exists(conf.res_file))
{
File.Delete(conf.res_file);
}
File.Move(tmp_res_file, conf.res_file);
conf.postproc.Tally();
return(0);
}
public int Exec(BuildConf conf)
{
var sw = new Stopwatch();
sw.Start();
int code = DoExec(conf);
sw.Stop();
Console.WriteLine("BHL build done({0} sec)", Math.Round(sw.ElapsedMilliseconds / 1000.0f, 2));
return(code);
}
bool WriteCompilationResultToFile(BuildConf conf, List <CompilerWorker> compiler_workers, string file_path)
{
using (FileStream dfs = new FileStream(file_path, FileMode.Create, System.IO.FileAccess.Write))
{
var mwriter = new MsgPack.MsgPackWriter(dfs);
mwriter.Write(ModuleLoader.COMPILE_FMT);
mwriter.Write(FILE_VERSION);
int total_modules = 0;
foreach (var w in compiler_workers)
{
if (w.error == null)
{
CheckUniqueSymbols(w);
}
//NOTE: exit in case of error
if (w.error != null)
{
if (conf.err_file == "-")
{
Console.Error.WriteLine(ErrorUtils.ToJson(w.error));
}
else
{
File.WriteAllText(conf.err_file, ErrorUtils.ToJson(w.error));
}
return(false);
}
total_modules += w.file2path.Count;
}
mwriter.Write(total_modules);
//NOTE: we'd like to write file binary modules in the same order they were added
for (int file_idx = 0; file_idx < conf.files.Count; ++file_idx)
{
var file = conf.files[file_idx];
foreach (var w in compiler_workers)
{
if (file_idx >= w.start && file_idx < w.start + w.count)
{
var path = w.file2path[file];
var compiled_file = w.file2compiled[file];
mwriter.Write((byte)conf.module_fmt);
mwriter.Write(path.name);
if (conf.module_fmt == ModuleBinaryFormat.FMT_BIN)
{
mwriter.Write(File.ReadAllBytes(compiled_file));
}
else if (conf.module_fmt == ModuleBinaryFormat.FMT_LZ4)
{
mwriter.Write(EncodeToLZ4(File.ReadAllBytes(compiled_file)));
}
else if (conf.module_fmt == ModuleBinaryFormat.FMT_FILE_REF)
{
mwriter.Write(compiled_file);
}
else
{
throw new Exception("Unsupported format: " + conf.module_fmt);
}
break;
}
}
}
return(true);
}
}
static List <CompilerWorker> StartAndWaitCompileWorkers(BuildConf conf, Types ts, List <ParseWorker> parse_workers)
{
var compiler_workers = new List <CompilerWorker>();
var cache = new Cache();
//1. waiting for parse workers to finish
foreach (var pw in parse_workers)
{
pw.Join();
//let's create the shared interim result
foreach (var kv in pw.file2interim)
{
cache.file2interim.Add(kv.Key, kv.Value);
}
}
bool had_errors = false;
foreach (var pw in parse_workers)
{
var cw = new CompilerWorker();
cw.id = pw.id;
cw.cache = cache;
cw.inc_dir = conf.inc_dir;
cw.cache_dir = pw.cache_dir;
cw.ts = ts;
cw.files = pw.files;
cw.start = pw.start;
cw.count = pw.count;
cw.postproc = conf.postproc;
//passing parser error if any
cw.error = pw.error;
if (pw.error != null)
{
had_errors = true;
}
compiler_workers.Add(cw);
}
//2. starting workers which now actually compile the code
// if there were no errors
if (!had_errors)
{
foreach (var w in compiler_workers)
{
w.Start();
}
foreach (var w in compiler_workers)
{
w.Join();
}
}
return(compiler_workers);
}
public void Run(string[] args)
{
var files = new List <string>();
string src_dir = "";
string res_file = "";
string tmp_dir = "";
bool use_cache = true;
string err_file = "";
string postproc_dll_path = "";
string userbindings_dll_path = "";
int max_threads = 1;
bool check_deps = true;
bool deterministic = false;
bool debug = false;
ModuleBinaryFormat module_fmt = ModuleBinaryFormat.FMT_LZ4;
var p = new OptionSet()
{
{ "dir=", "source dir",
v => src_dir = v },
{ "files=", "source files list",
v => files.AddRange(File.ReadAllText(v).Split(new string[] { "\r\n", "\n" }, StringSplitOptions.None)) },
{ "result=", "result file",
v => res_file = v },
{ "tmp-dir=", "tmp dir",
v => tmp_dir = v },
{ "C", "don't use cache",
v => use_cache = v == null },
{ "N", "don't check import deps",
v => check_deps = v == null },
{ "postproc-dll=", "posprocess dll path",
v => postproc_dll_path = v },
{ "bindings-dll=", "bindings dll path",
v => userbindings_dll_path = v },
{ "error=", "error file",
v => err_file = v },
{ "deterministic=", "deterministic build",
v => deterministic = v != null },
{ "threads=", "number of threads",
v => max_threads = int.Parse(v) },
{ "d", "debug version",
v => debug = v != null },
{ "module-fmt=", "binary module format",
v => module_fmt = (ModuleBinaryFormat)int.Parse(v) }
};
var extra = new List <string>();
try
{
extra = p.Parse(args);
}
catch (OptionException e)
{
Usage(e.Message);
}
files.AddRange(extra);
if (!Directory.Exists(src_dir))
{
Usage("Root source directory is not valid");
}
src_dir = Path.GetFullPath(src_dir);
if (res_file == "")
{
Usage("Result file path not set");
}
if (tmp_dir == "")
{
Usage("Tmp dir not set");
}
if (err_file == "")
{
Usage("Err file not set");
}
if (File.Exists(err_file))
{
File.Delete(err_file);
}
IUserBindings userbindings = new EmptyUserBindings();
if (userbindings_dll_path != "")
{
var userbindings_assembly = System.Reflection.Assembly.LoadFrom(userbindings_dll_path);
var userbindings_class = userbindings_assembly.GetTypes()[0];
userbindings = System.Activator.CreateInstance(userbindings_class) as IUserBindings;
if (userbindings == null)
{
Usage("User bindings are invalid");
}
}
IFrontPostProcessor postproc = new EmptyPostProcessor();
if (postproc_dll_path != "")
{
var postproc_assembly = System.Reflection.Assembly.LoadFrom(postproc_dll_path);
var postproc_class = postproc_assembly.GetTypes()[0];
postproc = System.Activator.CreateInstance(postproc_class) as IFrontPostProcessor;
if (postproc == null)
{
Usage("User postprocessor is invalid");
}
}
if (files.Count == 0)
{
Build.AddFilesFromDir(src_dir, files);
}
for (int i = files.Count; i-- > 0;)
{
if (string.IsNullOrEmpty(files[i]))
{
files.RemoveAt(i);
}
}
if (deterministic)
{
files.Sort();
}
Console.WriteLine("BHL total files: {0}(debug: {1}, cache: {2})", files.Count, debug, use_cache);
var conf = new BuildConf();
conf.args = string.Join(";", args);
conf.module_fmt = module_fmt;
conf.use_cache = use_cache;
conf.self_file = GetSelfFile();
conf.check_deps = check_deps;
conf.files = files;
conf.inc_dir = src_dir;
conf.max_threads = max_threads;
conf.res_file = res_file;
conf.tmp_dir = tmp_dir;
conf.err_file = err_file;
conf.userbindings = userbindings;
conf.postproc = postproc;
conf.debug = debug;
var build = new Build();
int err = build.Exec(conf);
if (err != 0)
{
Environment.Exit(err);
}
}