From c07183ec37be07656fc58799ce85fe4601dc5c86 Mon Sep 17 00:00:00 2001
From: Ralph Amissah <ralph.amissah@gmail.com>
Date: Fri, 14 Sep 2018 19:05:01 -0400
Subject: mess further with parallelism flags
---
src/doc_reform/doc_reform.d | 240 +++++++++++++++++++++++++++++++-------------
src/doc_reform/output/hub.d | 2 +-
2 files changed, 170 insertions(+), 72 deletions(-)
(limited to 'src')
diff --git a/src/doc_reform/doc_reform.d b/src/doc_reform/doc_reform.d
index 5ac96ec..2e581a0 100755
--- a/src/doc_reform/doc_reform.d
+++ b/src/doc_reform/doc_reform.d
@@ -70,7 +70,8 @@ void main(string[] args) {
"html-scroll" : false,
"manifest" : false,
"ocn" : true,
- "pp2" : false,
+ "parallelise" : true,
+ "parallelise-subprocesses" : false,
"quiet" : false,
"sisupod" : false,
"source" : false,
@@ -114,7 +115,8 @@ void main(string[] args) {
"html-scroll", "--html-seg process html output", &opts["html-scroll"],
"manifest", "--manifest process manifest output", &opts["manifest"],
"ocn", "--ocn object cite numbers (default)", &opts["ocn"],
- "pp2", "--pp2 nested parallelisation", &opts["pp2"],
+ "parallelise", "--parallelise parallelisation", &opts["parallelise"],
+ "parallelise-subprocesses", "--parallelise-subprocesses nested parallelisation", &opts["parallelise-subprocesses"],
"quiet", "--quiet output to terminal", &opts["quiet"],
"sisupod", "--sisupod sisupod source content bundled", &opts["sisupod"],
"source", "--source markup source text content", &opts["source"],
@@ -239,14 +241,29 @@ void main(string[] args) {
: false;
return _is;
}
- auto sqlite_insert() {
- return opts["sqlite-insert"];
- }
auto sqlite_delete() {
return opts["sqlite-delete"];
}
auto sqlite_update() {
- return opts["sqlite-update"];
+ bool _is = (
+ opts["sqlite-update"]
+ || opts["sqlite-insert"]
+ )
+ ? true
+ : false;
+ return _is;
+ }
+ auto sqlite_shared_db_action() {
+ bool _is = (
+ opts["sqlite-db-recreate"]
+ || opts["sqlite-db-create"]
+ || opts["sqlite-delete"]
+ || opts["sqlite-insert"]
+ || opts["sqlite-update"]
+ )
+ ? true
+ : false;
+ return _is;
}
auto text() {
return opts["text"];
@@ -302,8 +319,13 @@ void main(string[] args) {
auto sqlite_filename() {
return settings["sqlite-filename"];
}
- auto pp2() {
- return opts["pp2"];
+ auto parallelise() {
+ bool _is = ( opts["parallelise"]) ? true : false;
+ _is = ( sqlite_shared_db_action ) ? false : true;
+ return _is;
+ }
+ auto parallelise_subprocesses() {
+ return opts["parallelise-subprocesses"];
}
auto output_task_scheduler() {
int[] schedule;
@@ -341,7 +363,6 @@ void main(string[] args) {
|| source
|| sqlite_discrete
|| sqlite_delete
- || sqlite_insert
|| sqlite_update
)
? true
@@ -435,77 +456,154 @@ void main(string[] args) {
writeln("- step0 complete");
}
}
- if (_manifests.length > 1) { // _manifests[0] initialized dummy element
- foreach(manifest; parallel(_manifests[1..$])) {
- if (!empty(manifest.src.filename)) {
- scope(success) {
- if (!(_opt_action.quiet)) {
- writefln(
- "%s\n%s",
- "~ document complete, ok ~",
- "------------------------------------------------------------------",
- );
+ if (_manifests.length > 1 // _manifests[0] initialized dummy element
+ && _opt_action.abstraction) {
+ if (_opt_action.parallelise) { // note cannot parallelise sqlite shared db
+ foreach(manifest; parallel(_manifests[1..$])) {
+ if (!empty(manifest.src.filename)) {
+ scope(success) {
+ if (!(_opt_action.quiet)) {
+ writefln(
+ "%s\n%s",
+ "~ document complete, ok ~",
+ "------------------------------------------------------------------",
+ );
+ }
}
- }
- scope(failure) {
- debug(checkdoc) {
- stderr.writefln(
- "~ document run failure ~ (%s v%s)\n\t%s\n%s",
- __VENDOR__, __VERSION__,
- manifest.src.filename,
- "------------------------------------------------------------------",
- );
+ scope(failure) {
+ debug(checkdoc) {
+ stderr.writefln(
+ "~ document run failure ~ (%s v%s)\n\t%s\n%s",
+ __VENDOR__, __VERSION__,
+ manifest.src.filename,
+ "------------------------------------------------------------------",
+ );
+ }
}
- }
- enforce(
- manifest.src.filename.match(rgx.src_pth_types),
- "not a sisu markup filename: «" ~
- manifest.src.filename ~ "»"
- );
- debug(steps) {
- writeln("--->\nstepX commence → (document abstraction)");
- }
- auto t = DocReformAbstraction!()(_env, _opt_action, manifest);
- static assert(!isTypeTuple!(t));
- static assert(t.length==2);
- auto doc_abstraction = t[dAM.abstraction];
- auto doc_matters = t[dAM.matters];
- debug(steps) {
- writeln("- stepX complete");
- }
- /+ ↓ debugs +/
- if (doc_matters.opt.action.verbose) {
- DocReformAbstractionSummary!()(doc_abstraction, doc_matters);
- }
- /+ ↓ debugs +/
- if ((doc_matters.opt.action.debug_do)
- || (doc_matters.opt.action.verbose)
- ) {
- DocReformDebugs!()(doc_abstraction, doc_matters);
- }
- /+ ↓ output hub +/
- if (!(doc_matters.opt.action.skip_output)) {
+ enforce(
+ manifest.src.filename.match(rgx.src_pth_types),
+ "not a sisu markup filename: «" ~
+ manifest.src.filename ~ "»"
+ );
debug(steps) {
- writeln("step5 commence → (process outputs)");
+ writeln("--->\nstepX commence → (document abstraction)");
}
- outputHub!()(doc_abstraction, doc_matters);
+ auto t = DocReformAbstraction!()(_env, _opt_action, manifest);
+ static assert(!isTypeTuple!(t));
+ static assert(t.length==2);
+ auto doc_abstraction = t[dAM.abstraction];
+ auto doc_matters = t[dAM.matters];
debug(steps) {
- writeln("- step5 complete");
+ writeln("- stepX complete");
}
+ /+ ↓ debugs +/
+ if (doc_matters.opt.action.verbose) {
+ DocReformAbstractionSummary!()(doc_abstraction, doc_matters);
+ }
+ /+ ↓ debugs +/
+ if ((doc_matters.opt.action.debug_do)
+ || (doc_matters.opt.action.verbose)
+ ) {
+ DocReformDebugs!()(doc_abstraction, doc_matters);
+ }
+ /+ ↓ output hub +/
+ if (!(doc_matters.opt.action.skip_output)) {
+ debug(steps) {
+ writeln("step5 commence → (process outputs)");
+ }
+ outputHub!()(doc_abstraction, doc_matters);
+ debug(steps) {
+ writeln("- step5 complete");
+ }
+ }
+ scope(exit) {
+ if (!(_opt_action.quiet)) {
+ writefln(
+ "processed file: %s",
+ manifest.src.filename
+ );
+ }
+ destroy(manifest);
+ }
+ } else {
+ /+ no recognized filename provided +/
+ writeln("no recognized filename");
+ break; // terminate, stop
}
- scope(exit) {
- if (!(_opt_action.quiet)) {
- writefln(
- "processed file: %s",
- manifest.src.filename
- );
+ }
+ } else {
+ foreach(manifest; _manifests[1..$]) {
+ writeln("parallelisation off: actions include sqlite shared db");
+ if (!empty(manifest.src.filename)) {
+ scope(success) {
+ if (!(_opt_action.quiet)) {
+ writefln(
+ "%s\n%s",
+ "~ document complete, ok ~",
+ "------------------------------------------------------------------",
+ );
+ }
}
- destroy(manifest);
+ scope(failure) {
+ debug(checkdoc) {
+ stderr.writefln(
+ "~ document run failure ~ (%s v%s)\n\t%s\n%s",
+ __VENDOR__, __VERSION__,
+ manifest.src.filename,
+ "------------------------------------------------------------------",
+ );
+ }
+ }
+ enforce(
+ manifest.src.filename.match(rgx.src_pth_types),
+ "not a sisu markup filename: «" ~
+ manifest.src.filename ~ "»"
+ );
+ debug(steps) {
+ writeln("--->\nstepX commence → (document abstraction)");
+ }
+ auto t = DocReformAbstraction!()(_env, _opt_action, manifest);
+ static assert(!isTypeTuple!(t));
+ static assert(t.length==2);
+ auto doc_abstraction = t[dAM.abstraction];
+ auto doc_matters = t[dAM.matters];
+ debug(steps) {
+ writeln("- stepX complete");
+ }
+ /+ ↓ debugs +/
+ if (doc_matters.opt.action.verbose) {
+ DocReformAbstractionSummary!()(doc_abstraction, doc_matters);
+ }
+ /+ ↓ debugs +/
+ if ((doc_matters.opt.action.debug_do)
+ || (doc_matters.opt.action.verbose)
+ ) {
+ DocReformDebugs!()(doc_abstraction, doc_matters);
+ }
+ /+ ↓ output hub +/
+ if (!(doc_matters.opt.action.skip_output)) {
+ debug(steps) {
+ writeln("step5 commence → (process outputs)");
+ }
+ outputHub!()(doc_abstraction, doc_matters);
+ debug(steps) {
+ writeln("- step5 complete");
+ }
+ }
+ scope(exit) {
+ if (!(_opt_action.quiet)) {
+ writefln(
+ "processed file: %s",
+ manifest.src.filename
+ );
+ }
+ destroy(manifest);
+ }
+ } else {
+ /+ no recognized filename provided +/
+ writeln("no recognized filename");
+ break; // terminate, stop
}
- } else {
- /+ no recognized filename provided +/
- writeln("no recognized filename");
- break; // terminate, stop
}
}
}
diff --git a/src/doc_reform/output/hub.d b/src/doc_reform/output/hub.d
index 9e933d8..a0a73cd 100644
--- a/src/doc_reform/output/hub.d
+++ b/src/doc_reform/output/hub.d
@@ -52,7 +52,7 @@ template outputHub() {
msg.vv("html css & images done");
}
}
- if (!(doc_matters.opt.action.pp2)) {
+ if (!(doc_matters.opt.action.parallelise_subprocesses)) {
foreach(schedule; doc_matters.opt.action.output_task_scheduler) {
Scheduled!()(schedule, doc_abstraction, doc_matters);
}
--
cgit v1.2.3