From ce988ad5dc0398d721d9f4f7e0fd3f3dfdb41b38 Mon Sep 17 00:00:00 2001
From: "Silas S. Brown" <ssb22@cam.ac.uk>
Date: Wed, 4 Jan 2017 15:09:20 +0000
Subject: [PATCH] Update Annotator Generator, css-generate.py
git-svn-id: http://svn.code.sf.net/p/e-guidedog/code/ssb22/adjuster@2677 29193198-4895-4776-b068-10539e920549
---
annogen.py | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/annogen.py b/annogen.py
index 710fa73..59ea04d 100755
--- a/annogen.py
+++ b/annogen.py
@@ -240,6 +240,9 @@ parser.add_option("--time-estimate",
parser.add_option("--single-core",
action="store_true",default=False,
help="Use only one CPU core even when others are available. (If this option is not set, multiple cores are used if a 'futures' or 'mpi4py.futures' package is installed; this currently requires --checkpoint and is used only for large collocation checks in limited circumstances. MPI is not currently likely to achieve much speed increase, but concurrent.futures is.)") # (limited circumstances: namely, words that occur in length-1 phrases)
+parser.add_option("--force-mpi",
+ action="store_true",default=False,
+ help="For testing MPI setups, force use of MPI and print the traceback if it doesn't work, instead of silently falling back to non-MPI.")
main = (__name__ == "__main__")
if main: sys.stderr.write(program_name+"\n") # not sys.stdout: may or may not be showing --help (and anyway might want to process the help text for website etc)
@@ -253,6 +256,7 @@ import gc ; gc.disable() # should be OK if we don't create cycles (TODO: run gc.
def warn(msg):
if main: sys.stderr.write("Warning: "+msg+"\n")
# else it should have already been written
+if "PyPy" in sys.version: warn("PyPy is likely to run 60% slower than python with annogen") # (not to mention concurrent.futures being less likely to be available)
if primitive and ybytes: warn("primitive will override ybytes\n")
if ybytes: ybytes=int(ybytes)
@@ -2897,6 +2901,8 @@ def generate_map():
if checkpoint: pickle.Pickler(open(checkpoint+os.sep+'map','wb'),-1).dump((corpus_to_markedDown_map,c2m_inverse,precalc_sets,yPriorityDic))
checkpoint_exit()
+if force_mpi and single_core: errExit("You cannot specify --force-mpi with --single-core")
+if force_mpi and not checkpoint: errExit("--force-mpi currently requires --checkpoint")
def setup_parallelism():
if single_core or not checkpoint: return # parallelise only if checkpoint (otherwise could have trouble sharing the normalised corpus etc) TODO: document that checkpoint also affects this
try:
@@ -2906,7 +2912,8 @@ def setup_parallelism():
.index("-m mpi4py.futures") # ValueError if not found
import mpi4py.futures # mpi4py v2.1+
return mpi4py.futures.MPIPoolExecutor()
- except: pass
+ except:
+ if force_mpi: raise
try:
import concurrent.futures # sudo pip install futures (2.7 backport of 3.2 standard library)
import multiprocessing
--
GitLab