gradint-build/samples/utils/transliterate.py → samples/utils/transliterate.py

+#!/usr/bin/env python2
+
 # transliterate.py - print a 2nd-language-transliterated version of vocab.txt and any .txt pairs in samples
 # (may be useful for grepping, loading to Latin-only PDA, etc)
 # (note: leaves comments untransliterated, + may not translit all text if gradint is set up so a transliterating synth will not be used)

server/README.txt

+Gradint server tools
+--------------------
+
+gradint.cgi - CGI script to run Gradint completely online
+(but just synthesized vocabulary for now).
+Not integrated with email-lesson.sh below, because that needs a
+large vocab (which probably needs non web-based admin)
+
+email-lesson* - scripts that can help you to
+automatically distribute daily lessons to students
+using a web server with reminder emails
+
+vocab2html.py - make an HTML index for a synth cache,
+with the help of a vocab.txt (you can also use it with espeak.cgi)
+
+samples.cgi - CGI script to browse a samples directory
+(make sure you have permission to publish your recordings,
+or that the site is not publically viewable)
+
+espeak.cgi - script that lets a Web user play with espeak options
+
+Other files - see description at the top of the file

server/cantonese.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#  (should work with either Python 2 or Python 3)
+
+# cantonese.py - Python functions for processing Cantonese transliterations
+# (uses eSpeak and Gradint for help with some of them)
+
+# v1.48 (c) 2013-15,2017-24 Silas S. Brown.  License: GPL
+
+cache = {} # to avoid repeated eSpeak runs,
+# zi -> jyutping or (pinyin,) -> translit
+dryrun_mode = False # True = prepare to populate cache in batch
+jyutping_dryrun,pinyin_dryrun = set(),set()
+
+import re, pickle, os, sys
+if '--cache' in sys.argv:
+  cache_fname = sys.argv[sys.argv.index('--cache')+1]
+else: cache_fname = os.environ.get("JYUTPING_CACHE","/tmp/.jyutping-cache")
+try: cache = pickle.Unpickler(open(cache_fname,"rb")).load()
+except: pass
+
+extra_zhy_dict = { # TODO: add these to the real zhy_list in eSpeak
+  u"\u9c85":"bat3",u"\u9b81":"bat3",
+}
+
+def S(v): # make sure it's a string in both Python 2 and 3
+  if type("")==type(u""): # Python 3
+    try: return v.decode('utf-8') # in case it's bytes
+    except: return v
+  else: return v
+def B(v): # make sure it's bytes in Python 3, str in Python 2
+  if type(v)==type(u""): return v.encode('utf-8')
+  return v
+
+def get_jyutping(hanzi,mustWork=1):
+  if not type(hanzi)==type(u""): hanzi=hanzi.decode('utf-8')
+  for k,v in extra_zhy_dict.items(): hanzi=hanzi.replace(k,v)
+  global espeak
+  if not espeak:
+      espeak = import_gradint().ESpeakSynth()
+      if not espeak.works_on_this_platform(): # must call
+          raise Exception("espeak.works_on_this_platform")
+      assert espeak.supports_language("zhy")
+
+  global jyutping_dryrun
+  if dryrun_mode:
+      if not hanzi in cache: jyutping_dryrun.add(hanzi)
+      return "aai1" # placeholder value
+  elif jyutping_dryrun:
+      jyutping_dryrun = list(jyutping_dryrun)
+      vals = espeak.transliterate_multiple("zhy",jyutping_dryrun,0)
+      assert len(jyutping_dryrun)==len(vals)
+      for k,v in zip(jyutping_dryrun,vals):
+        cache[k]=S(v).replace("7","1").lower() # see below
+      jyutping_dryrun = set()
+  if hanzi in cache: jyutping = cache[hanzi]
+  else: cache[hanzi] = jyutping = S(espeak.transliterate("zhy",hanzi,forPartials=0)).replace("7","1").lower() # .lower() needed because espeak sometimes randomly capitalises e.g. 2nd hanzi of 'hypocrite' (Mandarin xuwei de ren)
+  if mustWork: assert jyutping.strip(), "No translit. result for "+repr(hanzi)
+  elif not jyutping.strip(): jyutping=""
+  return jyutping
+espeak = 0
+
+def hanzi_only(unitext): return u"".join(filter(lambda x:0x4e00<=ord(x)<0xa700 or ord(x)>=0x10000, list(unitext)))
+def py2nums(pinyin):
+  if not type(pinyin)==type(u""):
+    pinyin = pinyin.decode('utf-8')
+  if not pinyin.strip(): return ""
+  global pinyin_dryrun
+  if pinyin_dryrun:
+    pinyin_dryrun = list(pinyin_dryrun)
+    vals = espeak.transliterate_multiple("zh",pinyin_dryrun,0)
+    assert len(pinyin_dryrun)==len(vals)
+    for i in range(len(pinyin_dryrun)):
+      cache[(pinyin_dryrun[i],)]=vals[i]
+    pinyin_dryrun = set()
+  if (pinyin,) in cache: pyNums = cache[(pinyin,)]
+  else: pyNums = espeak.transliterate("zh",pinyin,forPartials=0) # (this transliterate just does tone marks to numbers, adds 5, etc; forPartials=0 because we DON'T want to change letters like X into syllables, as that won't happen in jyutping and we're going through it tone-by-tone)
+  assert pyNums and pyNums.strip(), "espeak.transliterate returned %s for %s" % (repr(pyNums),repr(pinyin))
+  return re.sub("a$","a5",re.sub("(?<=[a-zA-Z])er([1-5])",r"e\1r5",S(pyNums)))
+if type(u"")==type(""): # Python 3
+  getNext = lambda gen: gen.__next__()
+else: getNext = lambda gen: gen.next()
+def adjust_jyutping_for_pinyin(hanzi,jyutping,pinyin):
+  # If we have good quality (proof-read etc) Mandarin pinyin, this can sometimes improve the automatic Cantonese transcription
+  if not type(hanzi)==type(u""): hanzi = hanzi.decode('utf-8')
+  hanzi = hanzi_only(hanzi)
+  if not re.search(py2j_chars,hanzi): return jyutping
+  pinyin = re.findall('[A-Za-z]*[1-5]',py2nums(pinyin))
+  if not len(pinyin)==len(hanzi): return jyutping # can't fix
+  jyutping = S(jyutping)
+  i = 0 ; tones = re.finditer('[1-7]',jyutping) ; j2 = []
+  for h,p in zip(list(hanzi),pinyin):
+    try: j = getNext(tones).end()
+    except StopIteration: return jyutping # one of the hanzi has no Cantonese reading in our data: we'll warn "failed to fix" below
+    j2.append(jyutping[i:j]) ; i = j
+    if h in py2j and p.lower() in py2j[h]: j2[-1]=j2[-1][:re.search("[A-Za-z]*[1-7]$",j2[-1]).start()]+py2j[h][p.lower()]
+  return "".join(j2)+jyutping[i:]
+py2j={
+u"\u4E2D":{"zhong1":"zung1","zhong4":"zung3"},
+u"\u4E3A\u70BA":{"wei2":"wai4","wei4":"wai6"},
+u"\u4E50\u6A02":{"le4":"lok6","yue4":"ngok6"},
+u"\u4EB2\u89AA":{"qin1":"can1","qing4":"can3"},
+u"\u4EC0":{"shen2":"sam6","shi2":"sap6"}, # unless zaap6
+u"\u4F20\u50B3":{"chuan2":"cyun4","zhuan4":"zyun6"},
+u"\u4FBF":{"bian4":"bin6","pian2":"pin4"},
+u"\u5047":{"jia3":"gaa2","jia4":"gaa3"},
+u"\u5174\u8208":{"xing1":"hing1","xing4":"hing3"},
+# u"\u5207":{"qie4":"cai3","qie1":"cit3"}, # WRONG (rm'd v1.17).  It's cit3 in re4qie4.  It just wasn't in yiqie4 (which zhy_list has as an exception anyway)
+u"\u521B\u5275":{"chuang1":"cong1","chuang4":"cong3"},
+u"\u53EA":{"zhi1":"zek3","zhi3":"zi2"},
+u"\u53F7\u865F":{"hao4":"hou6","hao2":"hou4"},
+u"\u548C":{"he2":"wo4","he4":"wo6"},
+u"\u54BD":{"yan1":"jin1","yan4":"jin3","ye4":"jit3"},
+u"\u5708":{"juan4":"gyun6","quan1":"hyun1"},
+u"\u597D":{"hao3":"hou2","hao4":"hou3"},
+u"\u5C06\u5C07":{"jiang1":"zoeng1","jiang4":"zoeng3"},
+u"\u5C11":{"shao3":"siu2","shao4":"siu3"},
+u"\u5DEE":{"cha4":"caa1","cha1":"caa1","chai1":"caai1"},
+u"\u5F37\u5F3A":{"qiang2":"koeng4","qiang3":"koeng5"},
+u"\u62C5\u64D4":{"dan1":"daam1","dan4":"daam3"},
+u"\u6323\u6399":{"zheng4":"zaang6","zheng1":"zang1"},
+u"\u6570\u6578":{"shu3":"sou2","shu4":"sou3"},
+u"\u671D":{"chao2":"ciu4","zhao1":"ziu1"},
+u"\u6ED1":{"hua2":"waat6","gu3":"gwat1"},
+u"\u6F02":{"piao1":"piu1","piao3 piao4":"piu3"},
+u"\u76DB":{"sheng4":"sing6","cheng2":"sing4"},
+u"\u76F8":{"xiang1":"soeng1","xiang4":"soeng3"},
+u"\u770B":{"kan4":"hon3","kan1":"hon1"},
+u"\u79CD\u7A2E":{"zhong3":"zung2","zhong4":"zung3"},
+u"\u7EF7\u7E43":{"beng1":"bang1","beng3":"maang1"},
+u"\u8208":{"xing1":"hing1","xing4":"hing3"},
+u"\u843D":{"luo1 luo4 lao4":"lok6","la4":"laai6"},
+u"\u8457":{"zhu4":"zyu3","zhuo2":"zoek3","zhuo2 zhao2 zhao1 zhe5":"zoek6"},
+u"\u8981":{"yao4":"jiu3","yao1":"jiu1"},
+u"\u89C1\u898B":{"jian4":"gin3","xian4":"jin6"},
+u"\u89C9\u89BA":{"jue2":"gok3","jiao4":"gaau3"},
+u"\u8B58\u8BC6":{"shi2 shi4":"sik1","zhi4":"zi3"},
+u"\u8ABF\u8C03":{"diao4":"diu6","tiao2":"tiu4"},
+u"\u91CF":{"liang2":"loeng4","liang4":"loeng6"},
+u"\u9577\u957F":{"chang2":"coeng4","zhang3":"zoeng2"},
+u"\u9593\u95F4":{"jian1":"gaan1","jian4":"gaan3"},
+u"\u96BE\u96E3":{"nan2":"naan4","nan4":"naan6"}}
+for k in list(py2j.keys()):
+   if len(k)>1:
+    for c in list(k): py2j[c]=py2j[k]
+    del py2j[k]
+for _,v in py2j.items():
+  for k in list(v.keys()):
+    if len(k.split())>1:
+      for w in k.split(): v[w]=v[k]
+      del v[k]
+py2j_chars = re.compile(u'['+''.join(list(py2j.keys()))+']')
+
+def jyutping_to_lau(j):
+  j = S(j).lower().replace("j","y").replace("z","j")
+  for k,v in jlRep: j=j.replace(k,v)
+  return j.lower().replace("ohek","euk")
+def jyutping_to_lau_java(jyutpingNo=2,lauNo=1):
+  # for annogen.py 3.29+ --annotation-postprocess to ship Jyutping and generate Lau at runtime
+  return 'if(annotNo=='+str(jyutpingNo)+'||annotNo=='+str(lauNo)+'){m=Pattern.compile("<rt>(.*?)</rt>").matcher(r);sb=new StringBuffer();while(m.find()){String r2=(annotNo=='+str(jyutpingNo)+'?m.group(1).replaceAll("([1-7])(.)","$1&shy;$2"):(m.group(1)+" ").toLowerCase().replace("j","y").replace("z","j")'+''.join('.replace("'+k+'","'+v+'")' for k,v in jlRep)+'.toLowerCase().replace("ohek","euk").replaceAll("([1-7])","<sup>$1</sup>-").replace("- "," ").replaceAll(" $","")),tmp=m.group(1).substring(0,1);if(annotNo=='+str(lauNo)+'&&tmp.equals(tmp.toUpperCase()))r2=r2.substring(0,1).toUpperCase()+r2.substring(1);m.appendReplacement(sb,"<rt>"+r2+"</rt>");}m.appendTail(sb); r=sb.toString();}' # TODO: can probably go faster with mapping for some of this
+def incomplete_lau_to_jyutping(l):
+  # incomplete: assumes Lau didn't do the "aa" -> "a" rule
+  l = S(l).lower().replace("euk","ohek")
+  for k,v in ljRep: l=l.replace(k,v)
+  return l.lower().replace("j","z").replace("y","j")
+def incomplete_lau_to_yale_u8(l): return jyutping_to_yale_u8(incomplete_lau_to_jyutping(l))
+jlRep = [(unchanged,unchanged.upper()) for unchanged in "aai aau aam aang aan aap aat aak ai au am ang an ap at ak a ei eng ek e iu im ing in ip it ik i oi ong on ot ok ung uk".split()] + [("eoi","UI"),("eon","UN"),("eot","UT"),("eok","EUK"),("oeng","EUNG"),("oe","EUH"),("c","ch"),("ou","O"),("o","OH"),("yu","UE"),("u","OO")]
+jlRep.sort(key=lambda a:-len(a[0])) # longest 1st
+# u to oo includes ui to ooi, un to oon, ut to oot
+# yu to ue includes yun to uen and yut to uet
+# drawing from the table on http://www.omniglot.com/writing/cantonese.htm plus this private communication:
+# Jyutping "-oeng" maps to Sidney Lau "-eung".
+# Jyutping "jyu" maps to Sidney Lau "yue". (consequence of yu->ue, j->y)
+ljRep=[(b.lower(),a.upper()) for a,b in jlRep]
+ljRep.sort(key=lambda a:-len(a[0])) # longest 1st
+
+def ping_or_lau_to_syllable_list(j): return re.sub(r"([1-9])(?![0-9])",r"\1 ",re.sub(r"[!-/:-@^-`]"," ",S(j))).split()
+
+def hyphenate_ping_or_lau_syl_list(sList,groupLens=None):
+    if type(sList) in [str,type(u"")]:
+        sList = ping_or_lau_to_syllable_list(sList)
+    return hyphenate_syl_list(sList,groupLens)
+def hyphenate_yale_syl_list(sList,groupLens=None):
+    # (if sList is a string, the syllables must be space-separated,
+    #  which will be the case if to_yale functions below are used)
+    if not type(sList)==list: sList = sList.split()
+    return hyphenate_syl_list(sList,groupLens)
+def hyphenate_syl_list(sList,groupLens=None):
+    assert type(sList) == list
+    if '--hyphenate-all' in sys.argv: groupLens = [len(sList)]
+    elif not groupLens: groupLens = [1]*len(sList) # don't hyphenate at all if we don't know
+    else: assert sum(groupLens) == len(sList), "sum("+repr(groupLens)+")!=len("+repr(sList)+")"
+    r = [] ; start = 0
+    for g in groupLens:
+        r.append("-".join(S(x) for x in sList[start:start+g]))
+        start += g
+    return " ".join(r)
+    
+def jyutping_to_yale_TeX(j): # returns space-separated syllables
+  ret=[]
+  for syl in ping_or_lau_to_syllable_list(S(j).lower().replace("eo","eu").replace("oe","eu").replace("j","y").replace("yyu","yu").replace("z","j").replace("c","ch")):
+    vowel=lastVowel=None
+    for i in range(len(syl)):
+      if syl[i] in "aeiou":
+        vowel=i ; break
+    if vowel==None and re.match(r"h?(m|ng)[456]",syl): # standalone nasal syllables
+      vowel = syl.find('m')
+      if vowel<0: vowel = syl.index('n')
+      lastVowel = syl.find('g')
+      if lastVowel<0: lastVowel = vowel
+    if vowel==None:
+      ret.append(syl.upper()) ; continue # English word or letter in the Chinese?
+    if syl[vowel:vowel+2] == "aa" and (len(syl)<vowel+2 or syl[vowel+2] in "123456"):
+      syl=syl[:vowel]+syl[vowel+1:] # final aa -> a
+    # the tonal 'h' goes after all the vowels but before any consonants:
+    for i in range(len(syl)-1,-1,-1):
+      if syl[i] in "aeiou":
+        lastVowel=i ; break
+    if syl[-1] in "1234567":
+      # get_jyutping replaces 7 with 1 because zhy_list is
+      # more Canton-type than Hong Kong-type Cantonese and
+      # there is considerable disagreement on which "1"s
+      # should be "7"s, but if you pass any "7" into the
+      # jyutping_to_yale functions we can at least process
+      # it here:
+      tone = ["\=",r"\'","",r"\`",r"\'","",r"\`"][int(syl[-1])-1]
+      if syl[-1] in "456":
+        syl=syl[:lastVowel+1]+"h"+syl[lastVowel+1:]
+      ret.append((syl[:vowel]+tone+syl[vowel:-1]).replace(r"\=i",r"\=\i{}").replace(r"\=I",r"\=\I{}"))
+    else: ret.append(syl.upper()) # English word or letter in the Chinese?
+  return ' '.join(ret)
+
+def jyutping_to_yale_u8(j): # returns space-separated syllables
+  import unicodedata
+  def mysub(z,l):
+    for x,y in l:
+      z = re.sub(re.escape(x)+r"(.)",r"\1"+y,z)
+    return z
+  if type(u"")==type(""): U=str # Python 3
+  else: # Python 2
+    def U(x):
+      try: return x.decode('utf-8') # might be an emoji pass-through
+      except: return x # already Unicode
+  return unicodedata.normalize('NFC',mysub(U(jyutping_to_yale_TeX(j).replace(r"\i{}","i").replace(r"\I{}","I")),[(r"\`",u"\u0300"),(r"\'",u"\u0301"),(r"\=",u"\u0304")])).encode('utf-8')
+
+def superscript_digits_TeX(j):
+  # for jyutping and Sidney Lau
+  j = S(j)
+  for digit in "123456789": j=j.replace(digit,r"\raisebox{-0.3ex}{$^"+digit+r"$}\hspace{0pt}")
+  return j
+
+def superscript_digits_HTML(j):
+  j = S(j)
+  for digit in "123456789": j=j.replace(digit,"<sup>"+digit+"</sup>")
+  return j
+
+def superscript_digits_UTF8(j):
+  # WARNING: not all fonts have all digits; many have only the first 3.  superscript_digits_HTML might be better for browsers, even though it does produce more bytes.
+  j = S(j)
+  for digit in range(1,10): j=j.replace(str(digit),S(u"¹²³⁴⁵⁶⁷⁸⁹"[digit-1].encode('utf-8')))
+  if type(j)==type(u""): j=j.encode('utf-8') # Python 3
+  return j
+
+def import_gradint():
+    global gradint
+    try: return gradint
+    except: pass
+    # when importing gradint, make sure no command line
+    tmp,sys.argv = sys.argv,sys.argv[:1]
+    import gradint
+    sys.argv = tmp
+    gradint.espeak_preprocessors = {}
+    return gradint
+
+def do_song_subst(hanzi_u8): return B(hanzi_u8).replace(unichr(0x4f7f).encode('utf-8'),unichr(0x38c8).encode('utf-8')) # Mandarin shi3 (normally jyutping sai2) is usually si3 in songs, so substitute a rarer character that unambiguously has that reading before sending to get_jyutping
+
+if __name__ == "__main__":
+    # command-line use: output Lau for each line of stdin
+    # (or Yale if there's a --yale in sys.argv, or both
+    # with '#' separators if --yale#lau in sys.argv,
+    # also --yale#ping and --yale#lau#ping accepted);
+    # if there's a # in the line, assume it's hanzi#pinyin
+    # (for annogen.py --reannotator="##python cantonese.py")
+    lines = sys.stdin.read().replace("\r\n","\n").split("\n")
+    if lines and not lines[-1]: del lines[-1]
+    dryrun_mode = True
+    def songSubst(l):
+      if '--song-lyrics' in sys.argv: l=do_song_subst(l)
+      return l
+    for l in lines:
+      if '#' in l: l,pinyin = l.split('#')
+      else: pinyin = None
+      get_jyutping(songSubst(l))
+      if pinyin and not type(pinyin)==type(u""):
+        pinyin = pinyin.decode('utf-8')
+      if pinyin and not (pinyin,) in cache:
+        pinyin_dryrun.add(pinyin)
+        for w in pinyin.split():
+          for h in w.split('-'):
+            pinyin_dryrun.add(h)
+    dryrun_mode = False
+    for l in lines:
+      if '#' in l: l,pinyin = l.split('#')
+      else: pinyin = None
+      jyutping = get_jyutping(songSubst(l),0)
+      if not jyutping: groupLens = None # likely a Unihan-only 'fallback readings' zi that has no Cantonese
+      elif pinyin:
+        jyutping = adjust_jyutping_for_pinyin(l,jyutping,pinyin)
+        groupLens = [0]
+        for syl,space in re.findall('([A-Za-z]*[1-5])( *)',' '.join('-'.join(py2nums(h) for h in w.split('-')) for w in pinyin.split())): # doing it this way so we're not relying on espeak transliterate_multiple to preserve spacing and hyphenation
+          groupLens[-1] += 1
+          if space: groupLens.append(0)
+        if not groupLens[-1]: groupLens=groupLens[:-1]
+        lenWanted = len(ping_or_lau_to_syllable_list(jyutping))
+        if sum(groupLens) > lenWanted: # probably silent -r to drop
+          for i,word in enumerate(py2nums(pinyin).split()):
+            if re.search("[1-5]r5",word):
+              groupLens[i] -= 1
+              if sum(groupLens)==lenWanted: break
+        if not sum(groupLens)==lenWanted:
+          sys.stderr.write("WARNING: failed to fix "+pinyin+" ("+py2nums(pinyin)+") to "+jyutping+" ("+repr(ping_or_lau_to_syllable_list(jyutping))+") from "+l+", omitting\n")
+          groupLens = None ; jyutping = ""
+      else: groupLens = None
+      if "--yale#lau" in sys.argv: print (hyphenate_yale_syl_list(jyutping_to_yale_u8(jyutping),groupLens)+"#"+superscript_digits_HTML(hyphenate_ping_or_lau_syl_list(jyutping_to_lau(jyutping),groupLens)))
+      elif '--yale#ping' in sys.argv: print (hyphenate_yale_syl_list(jyutping_to_yale_u8(jyutping),groupLens)+"#"+jyutping.replace(' ',''))
+      elif "--yale#lau#ping" in sys.argv: print (hyphenate_yale_syl_list(jyutping_to_yale_u8(jyutping),groupLens)+"#"+superscript_digits_HTML(hyphenate_ping_or_lau_syl_list(jyutping_to_lau(jyutping),groupLens))+"#"+jyutping.replace(' ',''))
+      elif "--yale" in sys.argv: print (hyphenate_yale_syl_list(jyutping_to_yale_u8(jyutping),groupLens))
+      else: print (superscript_digits_HTML(hyphenate_ping_or_lau_syl_list(jyutping_to_lau(jyutping),groupLens)))
+    try: pickle.Pickler(open(cache_fname,"wb"),-1).dump(cache)
+    except: pass

gradint-build/samples/utils/email-lesson-archive.sh → server/email-lesson-archive.sh

 #!/bin/bash
 
 # email-lesson-archive.sh - archive an old email-lesson user
-# (C) 2008 Silas S. Brown, License: GPL
+# (C) 2008,2021-22 Silas S. Brown, License: GPL
 
 if ! pwd|grep email_lesson_users >/dev/null; then
   echo "This script should be run from an email_lesson_users directory (see email-lesson.sh)"
@@ -13,29 +13,29 @@ if test "a$1" == a; then
 fi
 . config
 while ! test "a$1" == a; do
-  if test -e "$1"; then
+  if [ -e "$1" ]; then
     unset U; unset Links
     if echo "$1"|grep "^user.0*" >/dev/null; then
       # specifying by user.0* id
       export U=$1
-      export Links=$(find . -maxdepth 1 -lname $U)
-    elif ls -l --color=none "$1"|grep ' -> ' >/dev/null; then
+      Links=$(find . -maxdepth 1 -lname "$U")
+    elif find "$1" -maxdepth 0 -type l|grep . >/dev/null; then
       # specifying by symlink
-      export Links=$1
-      export U=$(ls -l --color=none "$1"|sed -e 's/.* -> //')
+      Links=$1
+      U=$(ls -l --color=none "$1"|sed -e 's/.* -> //')
     else echo "Warning: can't make sense of username $1"; fi
-    if ! test "a$U" == a; then
-      if test -e $U/lastdate; then
+    if ! [ "a$U" == a ]; then
+      if [ -e "$U/lastdate" ]; then
         if test "a$Links" == a; then export Shortname=$U; else export Shortname=$Links; fi
-        if echo $PUBLIC_HTML | grep : >/dev/null; then
-        ssh $PUBLIC_HTML_EXTRA_SSH_OPTIONS $(echo $PUBLIC_HTML|sed -e 's/:.*//') rm -v $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$(cat $U/lastdate).*
-        else rm -v $PUBLIC_HTML/$U-$(cat $U/lastdate).*
+        if echo "$PUBLIC_HTML" | grep : >/dev/null; then
+        ssh $PUBLIC_HTML_EXTRA_SSH_OPTIONS "$(echo "$PUBLIC_HTML"|sed -e 's/:.*//')" rm -v "$(echo "$PUBLIC_HTML"|sed -e 's/[^:]*://')/$U-$(cat $U/lastdate).*"
+        else rm -v "$PUBLIC_HTML/$U-$(cat "$U/lastdate")".*
         fi
       fi
-      tar -jcvf $Shortname.tbz $U $Links
+      tar -jcvf "$Shortname.tbz" "$U" $Links
       mkdir -p old
-      mv -v --backup=numbered $Shortname.tbz old/
-      rm -rf $U $Links
+      mv -v --backup=numbered "$Shortname.tbz" old/
+      rm -rf "$U" $Links
     fi
   else echo "Warning: User $1 does not exist"; fi
 shift; done

gradint-build/samples/utils/email-lesson-report.sh → server/email-lesson-report.sh

@@ -6,7 +6,7 @@
 # - report is written to standard output so you can include
 # it in a script that makes some larger HTML page
 
-# v1.1 (C) 2007, 2009 Silas S. Brown, License: GPL
+# v1.12 (C) 2007, 2009. 2021-22 Silas S. Brown, License: GPL
 
 if ! pwd|grep email_lesson_users >/dev/null; then
   echo "This script should be run from an email_lesson_users directory (see email-lesson.sh)"
@@ -14,13 +14,13 @@ if ! pwd|grep email_lesson_users >/dev/null; then
 fi
 echo '<TABLE>'
 touch -d 0:00 /dev/shm/.midnight 2>/dev/null || touch -d 0:00 /tmp/.midnight
-if test -e /dev/shm/.midnight; then export Midnight=/dev/shm/.midnight; else export Midnight=/tmp/.midnight; fi
-for P in $(ls --color=none -t */progress.txt */podcasts-to-send 2>/dev/null); do
-  if test $P -nt $Midnight; then export Em="*";else unset Em; fi
-  if echo $P | grep podcasts-to-send$ >/dev/null; then
-    zgrep -H -m 1 . $P|grep -v ^user\.|sed -e 's,/.*:,</TD><TD COLSPAN=4>,' -e "s/^/<TR><TD>$Em/" -e "s,$,</TD></TR>,"
+if [ -e /dev/shm/.midnight ]; then Midnight=/dev/shm/.midnight; else Midnight=/tmp/.midnight; fi
+for P in $(ls --color=none -t -- */progress.txt */podcasts-to-send 2>/dev/null); do
+  if test "$P" -nt $Midnight; then Em="*";else unset Em; fi
+  if echo "$P" | grep podcasts-to-send$ >/dev/null; then
+    zgrep -H -m 1 . "$P"|grep -v ^user\.|sed -e 's,/.*:,</TD><TD COLSPAN=4>,' -e "s/^/<TR><TD>$Em/" -e "s,$,</TD></TR>,"
   else
-    zgrep -H -m 1 lessonsLeft $P|grep -v user\.|sed -e 's,/.*#,,' -e "s/^/<TR><TD>$Em/" -e "s, ,</TD><TD>,g" -e "s,$,</TD></TR>," -e "s/=/: /g"
+    zgrep -H -m 1 lessonsLeft "$P"|grep -v user\.|sed -e 's,/.*#,,' -e "s/^/<TR><TD>$Em/" -e "s, ,</TD><TD>,g" -e "s,$,</TD></TR>," -e "s/=/: /g"
   fi
 done
 rm $Midnight

gradint-build/samples/utils/email-lesson.sh → server/email-lesson.sh

@@ -3,114 +3,124 @@
 # email-lesson.sh: a script that can help you to
 # automatically distribute daily Gradint lessons
 # to students using a web server with reminder
-# emails.  Version 1.1128
+# emails.  Version 1.16
 
-# (C) 2007-2010 Silas S. Brown, License: GPL
+# (C) 2007-2010,2020-2022,2024 Silas S. Brown, License: GPL
 
-export DEFAULT_SUBJECT_LINE="Vocabulary practice (automatic message from gradint)"
-export DEFAULT_FORGOT_YESTERDAY="You forgot your lesson yesterday.
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+
+DEFAULT_SUBJECT_LINE="Vocabulary practice (automatic message from gradint)"
+DEFAULT_FORGOT_YESTERDAY="You forgot your lesson yesterday.
 Please remember to download your lesson from"
 # (NB include the words "you forgot" so that it's obvious this is a reminder not an additional lesson)
-export DEFAULT_EXPLAIN_FORGOT="Please try to hear one lesson every day.  If you download that lesson today,
+DEFAULT_EXPLAIN_FORGOT="Please try to hear one lesson every day.  If you download that lesson today,
 this program will make the next one for tomorrow."
-export DEFAULT_NEW_LESSON="Your lesson for today is at"
-export DEFAULT_LISTEN_TODAY="Please download and listen to it today."
-export DEFAULT_AUTO_MESSAGE="This is an automatic message from the gradint program.
+DEFAULT_NEW_LESSON="Your lesson for today is at"
+DEFAULT_LISTEN_TODAY="Please download and listen to it today."
+DEFAULT_AUTO_MESSAGE="This is an automatic message from the gradint program.
 Any problems, requests, or if you no longer wish to receive these emails,
 let me know."
 
-if ! test -e gradint.py; then
+if ! [ -e gradint.py ]; then
   echo "Error: This script should ALWAYS be run in the gradint directory."
   exit 1
 fi
 
-if which mail >/dev/null 2>/dev/null; then export DefaultMailProg=mail
-elif which mutt >/dev/null 2>/dev/null; then export DefaultMailProg="mutt -x"
-else export DefaultMailProg="ssh example.org mail"
+if which mail >/dev/null 2>/dev/null; then DefaultMailProg=mail
+elif which mutt >/dev/null 2>/dev/null; then DefaultMailProg="mutt -x"
+else DefaultMailProg="ssh example.org mail"
 fi
 
-if test "a$1" == "a--run"; then
+if [ "$1" == "--run" ]; then
   set -o pipefail # make sure errors in pipes are reported
-  if ! test -d email_lesson_users; then
+  if ! [ -d email_lesson_users ]; then
     echo "Error: script does not seem to have been set up yet"
     exit 1
   fi
-  export Gradint_Dir=$(pwd)
-  cd email_lesson_users
+  Gradint_Dir=$(pwd)
+  cd email_lesson_users || exit
   . config
-  if test -e $Gradint_Dir/.email-lesson-running; then
-    export Msg="Another email-lesson.sh --run is running - exitting.  (Remove $Gradint_Dir/.email-lesson-running if this isn't the case.)"
+  if [ -e "$Gradint_Dir/.email-lesson-running" ]; then
+    Msg="Another email-lesson.sh --run is running - exitting.  (Remove $Gradint_Dir/.email-lesson-running if this isn't the case.)"
     echo "$Msg"
     echo "$Msg"|$MailProg -s email-lesson-not-running $ADMIN_EMAIL # don't worry about retrying that
     exit 1
   fi
-  touch $Gradint_Dir/.email-lesson-running
-  if echo $PUBLIC_HTML | grep : >/dev/null && man ssh 2>/dev/null | grep ControlMaster >/dev/null; then
+  touch "$Gradint_Dir/.email-lesson-running"
+  if echo "$PUBLIC_HTML" | grep : >/dev/null && man ssh 2>/dev/null | grep ControlMaster >/dev/null; then
     # this version of ssh is new enough to support ControlPath, and PUBLIC_HTML indicates a remote host, so let's do it all through one connection
-    export ControlPath="-o ControlPath=$TMPDIR/__gradint_ctrl"
-    while true; do ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS -n -o ControlMaster=yes $ControlPath $(echo $PUBLIC_HTML|sed -e 's/:.*//') sleep 86400; sleep 10; done & export MasterPid=$!
+    ControlPath="-o ControlPath=$TMPDIR/__gradint_ctrl"
+    while true; do ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS -n -o ControlMaster=yes $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') sleep 86400; sleep 10; done & MasterPid=$!
   else unset MasterPid
   fi
-  (while ! bash -c "$CAT_LOGS_COMMAND"; do echo "cat-logs failed, re-trying in 61 seconds" 1>&2;sleep 61; done) | grep '/user\.' > $TMPDIR/._email_lesson_logs
+  (while ! bash -c "$CAT_LOGS_COMMAND"; do echo "cat-logs failed, re-trying in 61 seconds" >&2;sleep 61; done) | grep '/user\.' > "$TMPDIR/._email_lesson_logs"
   # (note: sleeping odd numbers of seconds so we can tell where it is if it gets stuck in one of these loops)
-  export Users="$(echo user.*)"
+  Users="$(echo user.*)"
   cd ..
   unset NeedRunMirror
   for U in $Users; do
     . email_lesson_users/config
-    if ! test "a$GLOBAL_GRADINT_OPTIONS" == a; then export GLOBAL_GRADINT_OPTIONS="$GLOBAL_GRADINT_OPTIONS ;"; fi
+    if [ "$GLOBAL_GRADINT_OPTIONS" ]; then GLOBAL_GRADINT_OPTIONS="$GLOBAL_GRADINT_OPTIONS ;"; fi
     # set some (but not all!) variables to defaults in case not set in profile
-    export SUBJECT_LINE="$DEFAULT_SUBJECT_LINE"
-    export FORGOT_YESTERDAY="$DEFAULT_FORGOT_YESTERDAY"
-    export LISTEN_TODAY="$DEFAULT_LISTEN_TODAY"
-    export NEW_LESSON="$DEFAULT_NEW_LESSON"
-    export EXPLAIN_FORGOT="$DEFAULT_EXPLAIN_FORGOT"
-    export AUTO_MESSAGE="$DEFAULT_AUTO_MESSAGE"
+    SUBJECT_LINE="$DEFAULT_SUBJECT_LINE"
+    FORGOT_YESTERDAY="$DEFAULT_FORGOT_YESTERDAY"
+    LISTEN_TODAY="$DEFAULT_LISTEN_TODAY"
+    NEW_LESSON="$DEFAULT_NEW_LESSON"
+    EXPLAIN_FORGOT="$DEFAULT_EXPLAIN_FORGOT"
+    AUTO_MESSAGE="$DEFAULT_AUTO_MESSAGE"
     unset Extra_Mailprog_Params1 Extra_Mailprog_Params2 GRADINT_OPTIONS
-    export Use_M3U=no
-    export FILE_TYPE=mp3
-    if grep $'\r' email_lesson_users/$U/profile >/dev/null; then
+    Use_M3U=no
+    FILE_TYPE=mp3
+    if grep $'\r' "email_lesson_users/$U/profile" >/dev/null; then
       # Oops, someone edited profile in a DOS line-endings editor (e.g. Wenlin on WINE for CJK stuff).  DOS line endings can mess up Extra_Mailprog_Params settings.
-      cat email_lesson_users/$U/profile | tr -d $'\r' > email_lesson_users/$U/profile.removeCR
-      mv email_lesson_users/$U/profile.removeCR email_lesson_users/$U/profile
+      tr -d $'\r' < "email_lesson_users/$U/profile" > email_lesson_users/$U/profile.removeCR
+      mv "email_lesson_users/$U/profile.removeCR" "email_lesson_users/$U/profile"
     fi
-    . email_lesson_users/$U/profile
-    if test "a$Use_M3U" == ayes; then export FILE_TYPE_2=m3u
-    else export FILE_TYPE_2=$FILE_TYPE; fi
+    . "email_lesson_users/$U/profile"
+    if [ "$Use_M3U" == yes ]; then FILE_TYPE_2=m3u
+    else FILE_TYPE_2=$FILE_TYPE; fi
     if echo "$MailProg" | grep ssh >/dev/null; then
       # ssh discards a level of quoting, so we need to be more careful
-      export SUBJECT_LINE="\"$SUBJECT_LINE\""
-      export Extra_Mailprog_Params1="\"$Extra_Mailprog_Params1\""
-      export Extra_Mailprog_Params2="\"$Extra_Mailprog_Params2\""
+      SUBJECT_LINE="\"$SUBJECT_LINE\""
+      Extra_Mailprog_Params1="\"$Extra_Mailprog_Params1\""
+      Extra_Mailprog_Params2="\"$Extra_Mailprog_Params2\""
     fi
-    if test -e email_lesson_users/$U/lastdate; then
-      if test "$(cat email_lesson_users/$U/lastdate)" == "$(date +%Y%m%d)"; then
+    if [ -e "email_lesson_users/$U/lastdate" ]; then
+      if [ "$(cat "email_lesson_users/$U/lastdate")" == "$(date +%Y%m%d)" ]; then
         # still on same day - do nothing with this user this time
 	continue
       fi
-      if ! grep $U-$(cat email_lesson_users/$U/lastdate)\. $TMPDIR/._email_lesson_logs >/dev/null
+      if ! grep "$U-$(cat email_lesson_users/$U/lastdate)"\. "$TMPDIR/._email_lesson_logs" >/dev/null
       # (don't add $FILE_TYPE after \. in case it has been changed)
       then
-        export Did_Download=0
-        if test -e email_lesson_users/$U/rollback; then
-          if test -e email_lesson_users/$U/progress.bak; then
-            mv email_lesson_users/$U/progress.bak email_lesson_users/$U/progress.txt
-            rm -f email_lesson_users/$U/progress.bin
-            export Did_Download=1 # (well actually they didn't, but we're rolling back)
+        Did_Download=0
+        if [ -e "email_lesson_users/$U/rollback" ]; then
+          if [ -e "email_lesson_users/$U/progress.bak" ]; then
+            mv "email_lesson_users/$U/progress.bak" "email_lesson_users/$U/progress.txt"
+            rm -f "email_lesson_users/$U/progress.bin"
+            Did_Download=1 # (well actually they didn't, but we're rolling back)
           fi # else can't rollback, as no progress.bak
-          if test -e email_lesson_users/$U/podcasts-to-send.old; then
-            mv email_lesson_users/$U/podcasts-to-send.old email_lesson_users/$U/podcasts-to-send
+          if [ -e "email_lesson_users/$U/podcasts-to-send.old" ]; then
+            mv "email_lesson_users/$U/podcasts-to-send.old" "email_lesson_users/$U/podcasts-to-send"
           fi
         fi
-      else export Did_Download=1; fi
-      rm -f email_lesson_users/$U/rollback
-      if test $Did_Download == 0; then
+      else Did_Download=1; fi
+      rm -f "email_lesson_users/$U/rollback"
+      if [ $Did_Download == 0 ]; then
         # send a reminder
-        export DaysOld="$(python -c "import os,time;print int((time.time()-os.stat('email_lesson_users/$U/lastdate').st_mtime)/3600/24)")"
-        if test $DaysOld -lt 5 || test $(date +%u) == 1; then # (remind only on Mondays if not checked for 5 days, to avoid filling up inboxes when people are away and can't get to email)
-        while ! $MailProg -s "$SUBJECT_LINE" $STUDENT_EMAIL "$Extra_Mailprog_Params1" "$Extra_Mailprog_Params2" <<EOF
+        DaysOld="$(python -c "import os,time;print(int((time.time()-os.stat('email_lesson_users/$U/lastdate').st_mtime)/3600/24))")"
+        if [ $DaysOld -lt 5 ] || [ $(date +%u) == 1 ]; then # (remind only on Mondays if not checked for 5 days, to avoid filling up inboxes when people are away and can't get to email)
+        while ! $MailProg -s "$SUBJECT_LINE" "$STUDENT_EMAIL" "$Extra_Mailprog_Params1" "$Extra_Mailprog_Params2" <<EOF
 $FORGOT_YESTERDAY
-$OUTSIDE_LOCATION/$U-$(cat email_lesson_users/$U/lastdate).$FILE_TYPE_2
+$OUTSIDE_LOCATION/$U-$(cat "email_lesson_users/$U/lastdate").$FILE_TYPE_2
 $EXPLAIN_FORGOT
 
 $AUTO_MESSAGE
@@ -119,68 +129,68 @@ do echo "mail sending failed; retrying in 62 seconds"; sleep 62; done; fi
         continue
       else
         # delete the previous lesson
-        if echo $PUBLIC_HTML | grep : >/dev/null; then ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo $PUBLIC_HTML|sed -e 's/:.*//') rm $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$(cat email_lesson_users/$U/lastdate).*
-        else rm $PUBLIC_HTML/$U-$(cat email_lesson_users/$U/lastdate).*; fi
+        if echo "$PUBLIC_HTML" | grep : >/dev/null; then ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') rm "$(echo "$PUBLIC_HTML"|sed -e 's/[^:]*://')/$U-$(cat "email_lesson_users/$U/lastdate").*"
+        else rm $PUBLIC_HTML/$U-$(cat "email_lesson_users/$U/lastdate").*; fi
 	# (.* because .$FILE_TYPE and possibly .m3u as well)
       fi
     fi
-    export CurDate=$(date +%Y%m%d)
-    if ! test "a$GRADINT_OPTIONS" == a; then export GRADINT_OPTIONS="$GRADINT_OPTIONS ;"; fi
-    if echo $PUBLIC_HTML | grep : >/dev/null; then export OUTDIR=$TMPDIR
-    else export OUTDIR=$PUBLIC_HTML; fi
-    export USER_GRADINT_OPTIONS="$GLOBAL_GRADINT_OPTIONS $GRADINT_OPTIONS samplesDirectory='email_lesson_users/$U/samples'; progressFile='email_lesson_users/$U/progress.txt'; pickledProgressFile='email_lesson_users/$U/progress.bin'; vocabFile='email_lesson_users/$U/vocab.txt';saveLesson='';loadLesson=0;progressFileBackup='email_lesson_users/$U/progress.bak';outputFile="
+    CurDate=$(date +%Y%m%d)
+    if [ "$GRADINT_OPTIONS" ]; then GRADINT_OPTIONS="$GRADINT_OPTIONS ;"; fi
+    if echo "$PUBLIC_HTML" | grep : >/dev/null; then OUTDIR=$TMPDIR
+    else OUTDIR=$PUBLIC_HTML; fi
+    USER_GRADINT_OPTIONS="$GLOBAL_GRADINT_OPTIONS $GRADINT_OPTIONS samplesDirectory='email_lesson_users/$U/samples'; progressFile='email_lesson_users/$U/progress.txt'; pickledProgressFile='email_lesson_users/$U/progress.bin'; vocabFile='email_lesson_users/$U/vocab.txt';saveLesson='';loadLesson=0;progressFileBackup='email_lesson_users/$U/progress.bak';outputFile="
     # (note: we DO keep progressFileBackup, because it can be useful if the server goes down and the MP3's need to be re-generated or something)
     unset Send_Podcast_Instead
-    if test -s email_lesson_users/$U/podcasts-to-send; then
-      export Send_Podcast_Instead="$(head -1 email_lesson_users/$U/podcasts-to-send)"
-      export NumLines=$(echo $(cat email_lesson_users/$U/podcasts-to-send|wc -l)-1|bc)
-      tail -$NumLines email_lesson_users/$U/podcasts-to-send > email_lesson_users/$U/podcasts-to-send2
-      mv email_lesson_users/$U/podcasts-to-send email_lesson_users/$U/podcasts-to-send.old
-      mv email_lesson_users/$U/podcasts-to-send2 email_lesson_users/$U/podcasts-to-send
-      if test $NumLines == 0; then
-        echo $U | $MailProg -s Warning:email-lesson-run-out-of-podcasts $ADMIN_EMAIL
+    if [ -s "email_lesson_users/$U/podcasts-to-send" ]; then
+      Send_Podcast_Instead="$(head -1 email_lesson_users/$U/podcasts-to-send)"
+      NumLines=$[$(cat "email_lesson_users/$U/podcasts-to-send"|wc -l)-1]
+      tail -$NumLines "email_lesson_users/$U/podcasts-to-send" > "email_lesson_users/$U/podcasts-to-send2"
+      mv "email_lesson_users/$U/podcasts-to-send" "email_lesson_users/$U/podcasts-to-send.old"
+      mv "email_lesson_users/$U/podcasts-to-send2" "email_lesson_users/$U/podcasts-to-send"
+      if [ $NumLines == 0 ]; then
+        echo "$U" | $MailProg -s Warning:email-lesson-run-out-of-podcasts $ADMIN_EMAIL
       fi
-    else rm -f email_lesson_users/$U/podcasts-to-send.old # won't be a rollback after this
+    else rm -f "email_lesson_users/$U/podcasts-to-send.old" # won't be a rollback after this
     fi
-    if test "$ENCODE_ON_REMOTE_HOST" == 1; then
-      export ToSleep=123
-      while ! if test "a$Send_Podcast_Instead" == a; then
-        python gradint.py "$USER_GRADINT_OPTIONS '-.sh'" </dev/null 2>$TMPDIR/__stderr | ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo $PUBLIC_HTML|sed -e 's/:.*//') "mkdir -p $REMOTE_WORKING_DIR; cd $REMOTE_WORKING_DIR; cat > __gradint.sh;chmod +x __gradint.sh;PATH=$SOX_PATH ./__gradint.sh|$ENCODING_COMMAND $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.$FILE_TYPE;rm -f __gradint.sh";
+    if [ "$ENCODE_ON_REMOTE_HOST" == 1 ]; then
+      ToSleep=123
+      while ! if [ ! "$Send_Podcast_Instead" ]; then
+        python gradint.py "$USER_GRADINT_OPTIONS '-.sh'" </dev/null 2>"$TMPDIR/__stderr" | ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') "mkdir -p $REMOTE_WORKING_DIR; cd $REMOTE_WORKING_DIR; cat > __gradint.sh;chmod +x __gradint.sh;PATH=$SOX_PATH ./__gradint.sh|$ENCODING_COMMAND $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.$FILE_TYPE;rm -f __gradint.sh";
       else
-        cd email_lesson_users/$U ; cat "$Send_Podcast_Instead" | ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo $PUBLIC_HTML|sed -e 's/:.*//') "cat > $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.$FILE_TYPE"; cd ../..;
+        cd "email_lesson_users/$U" ; cat "$Send_Podcast_Instead" | ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') "cat > $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.$FILE_TYPE"; cd ../..;
       fi; do
         # (</dev/null so exceptions don't get stuck on 'press enter to continue' to a temp stderr if running from a terminal)
-        $MailProg -s gradint-to-ssh-failed,-will-retry $ADMIN_EMAIL < $TMPDIR/__stderr
+        $MailProg -s gradint-to-ssh-failed,-will-retry $ADMIN_EMAIL < "$TMPDIR/__stderr"
         # (no spaces in subj so no need to decide whether to single or double quote)
         # (don't worry about mail errors - if net is totally down that's ok, admin needs to know if it's a gradint bug causing infinite loop)
-        sleep $ToSleep ; export ToSleep=$(echo $ToSleep*1.5|bc) # (increasing-time retries)
+        sleep $ToSleep ; ToSleep=$[$ToSleep*1.5] # (increasing-time retries)
       done
-      rm $TMPDIR/__stderr
-      if test "a$Use_M3U" == ayes; then
-        while ! ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo $PUBLIC_HTML|sed -e 's/:.*//') "echo $OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE > $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.m3u"; do sleep 63; done
+      rm "$TMPDIR/__stderr"
+      if [ "$Use_M3U" == yes ]; then
+        while ! ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') "echo $OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE > $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.m3u"; do sleep 63; done
       fi
     else # not ENCODE_ON_REMOTE_HOST
-      if ! test "a$Send_Podcast_Instead" == a; then
-        (cd email_lesson_users/$U ; cat "$Send_Podcast_Instead") > "$OUTDIR/$U-$CurDate.$FILE_TYPE"
+      if [ "$Send_Podcast_Instead" ]; then
+        (cd "email_lesson_users/$U" ; cat "$Send_Podcast_Instead") > "$OUTDIR/$U-$CurDate.$FILE_TYPE"
       elif ! python gradint.py "$USER_GRADINT_OPTIONS '$OUTDIR/$U-$CurDate.$FILE_TYPE'" </dev/null; then
         echo "Errors from gradint itself (not ssh/network); skipping this user."
         echo "Failed on $U, check output " | $MailProg -s gradint-failed $ADMIN_EMAIL
         continue
       fi
-      if test "a$Use_M3U" == ayes; then
-        echo $OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE > $OUTDIR/$U-$CurDate.m3u
+      if [ "$Use_M3U" == yes ]; then
+        echo "$OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE" > "$OUTDIR/$U-$CurDate.m3u"
       fi
-      if echo $PUBLIC_HTML | grep : >/dev/null; then
-        while ! scp $ControlPath -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $OUTDIR/$U-$CurDate.* $PUBLIC_HTML/; do
+      if echo "$PUBLIC_HTML" | grep : >/dev/null; then
+        while ! scp $ControlPath -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $OUTDIR/$U-$CurDate.* "$PUBLIC_HTML/"; do
           echo "scp failed; re-trying in 60 seconds"
   	sleep 64
         done
-        rm $OUTDIR/$U-$CurDate.*
+        rm "$OUTDIR/$U-$CurDate".*
       fi
     fi
-    export NeedRunMirror=1
-    if ! test -e email_lesson_users/$U/progress.bak; then touch email_lesson_users/$U/progress.bak; fi # so rollback works after 1st lesson
-    while ! $MailProg -s "$SUBJECT_LINE" $STUDENT_EMAIL "$Extra_Mailprog_Params1" "$Extra_Mailprog_Params2" <<EOF
+    NeedRunMirror=1
+    if ! [ -e "email_lesson_users/$U/progress.bak" ]; then touch "email_lesson_users/$U/progress.bak"; fi # so rollback works after 1st lesson
+    while ! $MailProg -s "$SUBJECT_LINE" "$STUDENT_EMAIL" "$Extra_Mailprog_Params1" "$Extra_Mailprog_Params2" <<EOF
 $NEW_LESSON
 $OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE_2
 $LISTEN_TODAY
@@ -188,27 +198,27 @@ $LISTEN_TODAY
 $AUTO_MESSAGE
 EOF
 do echo "mail sending failed; retrying in 65 seconds"; sleep 65; done
-    echo $CurDate > email_lesson_users/$U/lastdate
+    echo "$CurDate" > "email_lesson_users/$U/lastdate"
     unset AdminNote
-    if test "a$Send_Podcast_Instead" == a; then
-      if test $(zgrep -H -m 1 lessonsLeft email_lesson_users/$U/progress.txt|sed -e 's/.*=//') == 0; then export AdminNote="Note: $U has run out of new words"; fi
-    elif ! test -e email_lesson_users/$U/podcasts-to-send; then export AdminNote="Note: $U has run out of podcasts"; fi
-    if ! test "a$AdminNote" == a; then
-      while ! echo "$AdminNote"|$MailProg -s gradint-user-ran-out $ADMIN_EMAIL; do echo "Mail sending failed; retrying in 67 seconds"; sleep 67; done
+    if [ "$Send_Podcast_Instead" == a ]; then
+      if [ "$(zgrep -H -m 1 lessonsLeft "email_lesson_users/$U/progress.txt"|sed -e 's/.*=//')" == 0 ]; then AdminNote="Note: $U has run out of new words"; fi
+    elif ! [ -e "email_lesson_users/$U/podcasts-to-send" ]; then AdminNote="Note: $U has run out of podcasts"; fi
+    if [ "$AdminNote" ]; then
+      while ! echo "$AdminNote"|$MailProg -s gradint-user-ran-out "$ADMIN_EMAIL"; do echo "Mail sending failed; retrying in 67 seconds"; sleep 67; done
     fi
   done # end of per-user loop
-  if test "a$NeedRunMirror" == "a1" && ! test "a$PUBLIC_HTML_MIRROR_COMMAND" == a; then
+  if [ "$NeedRunMirror" == "1" ] && [ "$PUBLIC_HTML_MIRROR_COMMAND" ]; then
     while ! $PUBLIC_HTML_MIRROR_COMMAND; do
       echo "PUBLIC_HTML_MIRROR_COMMAND failed; retrying in 79 seconds"
-      echo As subject | $MailProg -s "PUBLIC_HTML_MIRROR_COMMAND failed, will retry" $ADMIN_EMAIL || true # ignore errors
+      echo As subject | $MailProg -s "PUBLIC_HTML_MIRROR_COMMAND failed, will retry" "$ADMIN_EMAIL" || true # ignore errors
       sleep 79
     done
   fi
-  rm -f $TMPDIR/._email_lesson_logs
-  if ! test a$MasterPid == a; then
+  rm -f "$TMPDIR/._email_lesson_logs"
+  if [ $MasterPid ] ; then
     kill $MasterPid
-    kill $(ps axwww|grep $TMPDIR/__gradint_ctrl|sed -e 's/^ *//' -e 's/ .*//') 2>/dev/null
-    rm -f $TMPDIR/__gradint_ctrl # in case ssh doesn't
+    kill $(pgrep -f "$TMPDIR/__gradint_ctrl") 2>/dev/null
+    rm -f "$TMPDIR/__gradint_ctrl" # in case ssh doesn't
   fi
   rm -f "$Gradint_Dir/.email-lesson-running"
   exit 0
@@ -217,10 +227,10 @@ fi
 echo "After setting up users, run this script daily with --run on the command line."
 echo "As --run was not specified, it will now go into setup mode."
 # Setup:
-if test "a$EDITOR" == a; then
+if ! [ "$EDITOR" ]; then
   echo "Error: No EDITOR environment variable set"; exit 1
 fi
-if ! test -e email_lesson_users/config; then
+if ! [ -e email_lesson_users/config ]; then
  echo "It seems the email_lesson_users directory is not set up"
  echo "Press Enter to create a new one,
  or Ctrl-C to quit if you're in the wrong directory"
@@ -228,11 +238,11 @@ if ! test -e email_lesson_users/config; then
  mkdir email_lesson_users || exit 1
  cat > email_lesson_users/config <<EOF
 # You need to edit this file.
-export GLOBAL_GRADINT_OPTIONS="" # if set, will be added to all gradint command lines (e.g. to set synthCache if it's not in advanced.txt)
-export MailProg="$DefaultMailProg" # mail, or mutt -x, or ssh some.host mail, or whatever
-export PUBLIC_HTML=~/public_html # where to put files on the WWW.  If it contains a : then scp will be used to copy them there.
-export OUTSIDE_LOCATION=http://$(hostname -f)/~$(whoami) # where they appear from outside
-export CAT_LOGS_COMMAND="false" # Please change this to a command that cats the
+GLOBAL_GRADINT_OPTIONS="" # if set, will be added to all gradint command lines (e.g. to set synthCache if it's not in advanced.txt)
+MailProg="$DefaultMailProg" # mail, or mutt -x, or ssh some.host mail, or whatever
+PUBLIC_HTML=~/public_html # where to put files on the WWW.  If it contains a : then scp will be used to copy them there.
+OUTSIDE_LOCATION=http://$(hostname -f)/~$(whoami) # where they appear from outside
+CAT_LOGS_COMMAND="false" # Please change this to a command that cats the
 # server logs for at least the last 48 hours.  (On some systems you may need
 # to make the script suid root.)  It is used to check that the users have
 # downloads their lessons and remind them if not.
@@ -244,29 +254,29 @@ export CAT_LOGS_COMMAND="false" # Please change this to a command that cats the
 # control connection (\$ControlPath will expand to
 # nothing on systems with old ssh's that don't support this)
 
-export PUBLIC_HTML_EXTRA_SSH_OPTIONS="" # if set and PUBLIC_HTML is on a remote host, these options will be added to all ssh and scp commands to that host - use this for things like specifying an alternative identity file with -i
+PUBLIC_HTML_EXTRA_SSH_OPTIONS="" # if set and PUBLIC_HTML is on a remote host, these options will be added to all ssh and scp commands to that host - use this for things like specifying an alternative identity file with -i
 
-export PUBLIC_HTML_MIRROR_COMMAND="" # if set, will be run after any new lessons are written to PUBLIC_HTML.
+PUBLIC_HTML_MIRROR_COMMAND="" # if set, will be run after any new lessons are written to PUBLIC_HTML.
 # This is for unusual setups where PUBLIC_HTML is not the real public_html directory but some command can be run to mirror its contents to the real one (perhaps on a remote server that cannot take passwordless SSH from here; of course you'd need to set up an alternative way of getting the files across and the log entries back).
 # Note: Do not add >/dev/null or similar redirects to PUBLIC_HTML_MIRROR_COMMAND as some versions of bash will give an error.
 
 export TMPDIR=/tmp # or /dev/shm or whatever
 
-export ENCODE_ON_REMOTE_HOST=0  # if 1, will ssh to the remote host
+ENCODE_ON_REMOTE_HOST=0  # if 1, will ssh to the remote host
 # that's specified in PUBLIC_HTML (which *must* be host:path in this case)
 # and will run an encoding command *there*, instead of encoding
 # locally and copying up.  This is useful if the local machine is the
 # only place gradint can run but it can't encode (e.g. Linux server running on NAS device).
 # If you set the above to 1 then you also need to set these options:
-export REMOTE_WORKING_DIR=. # directory to change to on remote host e.g. /tmp/gradint (will create with mkdir -p if does not exist)
+REMOTE_WORKING_DIR=. # directory to change to on remote host e.g. /tmp/gradint (will create with mkdir -p if does not exist)
 # (make sure $PUBLIC_HTML etc is absolute or is relative to $REMOTE_WORKING_DIR) (don't use spaces in these pathnames)
-export SOX_PATH=$PATH
+SOX_PATH=$PATH
 # make sure the above includes the remote host's "sox" as well as basic commands
-export ENCODING_COMMAND="lame --vbr-new -V 9 -"
+ENCODING_COMMAND="lame --vbr-new -V 9 -"
 # (used only if ENCODE_ON_REMOTE_HOST is set)
 # (include the full path for that if necessary; SOX_PATH will NOT be searched)
 # (set options for encode wav from stdin & output to the file specified on nxt parameter.  No shell quoting.)
-export ADMIN_EMAIL=admin@example.com # to report errors
+ADMIN_EMAIL=admin@example.com # to report errors
 EOF
  cd email_lesson_users; $EDITOR config; cd ..
  echo "Created email_lesson_users/config"
@@ -275,16 +285,16 @@ cd email_lesson_users
 while true; do
   echo "Type a user alias (or just press Enter) to add a new user, or Ctrl-C to quit"
   read Alias
-  export ID=$(mktemp -d user.$(python -c 'import random; print random.random()')XXXXXX) # (newer versions of mktemp allow more than 6 X's so the python step isn't necessary, but just in case we want to make sure that it's hard to guess the ID)
-  if ! test "a$Alias" == a; then ln -s $ID "$Alias"; fi
-  cd $ID
+  ID=$(mktemp -d user.$(python -c 'import random; print(random.random())')XXXXXX) # (newer versions of mktemp allow more than 6 X's so the python step isn't necessary, but just in case we want to make sure that it's hard to guess the ID)
+  if [ "$Alias" ]; then ln -s "$ID" "$Alias"; fi
+  cd "$ID" || exit 1
   cat > profile <<EOF
 # You need to edit the settings in this file.
-export STUDENT_EMAIL=student@example.org  # change to student's email address
+STUDENT_EMAIL=student@example.org  # change to student's email address
 export GRADINT_OPTIONS="" # extra gradint command-line options, for example to
                           # specify a different first and second language
-export FILE_TYPE=mp3 # change to something else if you want
-export Use_M3U=no # if yes, sends a .m3u link to the student
+FILE_TYPE=mp3 # change to something else if you want
+Use_M3U=no # if yes, sends a .m3u link to the student
 # instead of sending the file link directly.  Use this if
 # the student needs to stream over a slow link, but note
 # that it makes offline listening one step more complicated.
@@ -314,33 +324,33 @@ export Use_M3U=no # if yes, sends a .m3u link to the student
 # students that the lessons will now come from a different address.
 
 # Optional settings for customising the text of the message:
-export SUBJECT_LINE="$DEFAULT_SUBJECT_LINE"
-export FORGOT_YESTERDAY="$DEFAULT_FORGOT_YESTERDAY"
-export LISTEN_TODAY="$DEFAULT_LISTEN_TODAY"
-export NEW_LESSON="$DEFAULT_NEW_LESSON"
-export EXPLAIN_FORGOT="$DEFAULT_EXPLAIN_FORGOT"
-export AUTO_MESSAGE="$DEFAULT_AUTO_MESSAGE"
-export Extra_Mailprog_Params1=""
-export Extra_Mailprog_Params2=""
+SUBJECT_LINE="$DEFAULT_SUBJECT_LINE"
+FORGOT_YESTERDAY="$DEFAULT_FORGOT_YESTERDAY"
+LISTEN_TODAY="$DEFAULT_LISTEN_TODAY"
+NEW_LESSON="$DEFAULT_NEW_LESSON"
+EXPLAIN_FORGOT="$DEFAULT_EXPLAIN_FORGOT"
+AUTO_MESSAGE="$DEFAULT_AUTO_MESSAGE"
+Extra_Mailprog_Params1=""
+Extra_Mailprog_Params2=""
 # You may need to set Extra_Mailprog_Params to extra parameters
 # if the subject or text includes characters that need to be sent
-# in a specific charset.  For example, to send in Chinese GB2312
-# with Mutt, you can do this:
+# in a specific charset.  For example, to send Chinese (Simplified)
+# in UTF-8 with Mutt, you can do this:
 # export GRADINT_OPTIONS="firstLanguage='zh'; secondLanguage='en'; otherLanguages=[]"
 # export LANG=C
-# export Extra_Mailprog_Params1="-e"
-# export Extra_Mailprog_Params2="set charset='gb2312'; set send_charset='gb2312'"
-# export SUBJECT_LINE="英文词汇练习 (English vocabulary practice)"
-# export FORGOT_YESTERDAY="你忘记了昨天的课 (you forgot your lesson yesterday).
-# 请记得下载 (please remember to download) :"
-# export EXPLAIN_FORGOT="请试图天天听一课 (please try to hear one lesson every day)
-# 如果你今天下载, 这个软件要明天给你另一个课.
+# Extra_Mailprog_Params1="-e"
+# Extra_Mailprog_Params2="set charset='utf-8'; set send_charset='utf-8'"
+# SUBJECT_LINE="英文词汇练习 (English vocabulary practice)"
+# FORGOT_YESTERDAY="你忘记了昨天的课 (you forgot your lesson yesterday).
+# 请记得下载 (please remember to download) :"
+# EXPLAIN_FORGOT="请试图天天听一课 (please try to hear one lesson every day)
+# 如果你今天下载, 这个软件要明天给你另一个课.
 # (If you download that lesson today,
 # this program will make the next one for tomorrow.)"
-# export NEW_LESSON="今天的课在以下的网址 (your lesson for today is at)"
-# export LISTEN_TODAY="请你今天下载而听 (please download and listen to it today)."
-# export AUTO_MESSAGE="这个电邮是软件写的 (this is an automatic message from the gradint program).
-# 假如你有问题, 请告诉我 (any problems, let me know)."
+# NEW_LESSON="今天的课在以下的网址 (your lesson for today is at)"
+# LISTEN_TODAY="请你今天下载而听 (please download and listen to it today)."
+# AUTO_MESSAGE="这个电邮是软件写的 (this is an automatic message from the gradint program).
+# 假如你有问题, 请告诉我 (any problems, let me know)."
 
 # You can also override *some* of the email_lesson_users/config
 # options on a per-user basis by putting them here,

gradint-build/samples/utils/espeak.cgi → server/espeak.cgi

 #!/usr/bin/env python
+#  (Python 2 or Python 3)
 
 # espeak.cgi - a CGI script for the eSpeak speech synthesizer
 
-# (c) 2008 Silas S. Brown, License: GPL
-version="1.1211"
+# (c) 2008,2011,2020 Silas S. Brown, License: GPL
+version="1.3"
+
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
 
 # With most webservers you should be able to put this
 # in your public_html and do chmod +x.  You will also need to
@@ -21,15 +32,28 @@ max_input_size = 1000
 
 minSpeed, defaultSpeed, maxSpeed, speedStep = 80,170,370,30 # (NB check defaultSpeed=minSpeed+integer*speedStep)
 
-import os,commands,re,sys
-if commands.getoutput("which espeak 2>/dev/null"): prog="espeak"
-elif commands.getoutput("which speak 2>/dev/null"): prog="speak"
+import os,re,sys
+try: from commands import getoutput # Python 2
+except: from subprocess import getoutput # Python 3
+if getoutput("which espeak 2>/dev/null"): prog="espeak"
+elif getoutput("which speak 2>/dev/null"): prog="speak"
 else: raise Exception("Cannot find espeak")
 
-lang = f.getfirst("l",default_language)
+def S(x):
+    if type("")==type(u""): # Python 3
+        try: return x.decode('utf-8') # in case byte-string
+        except: pass
+    return x
+lang = S(f.getfirst("l",default_language))
 if len(lang)>10 or not re.match("^[a-z0-9-+]*$",lang): lang=default_language
 
-voiceDir=os.environ.get("ESPEAK_DATA_PATH","/usr/share/espeak-data")+"/voices"
+if "ESPEAK_DATA_PATH" in os.environ: voiceDir = os.environ["ESPEAK_DATA_PATH"]
+elif os.path.exists("/usr/share/espeak-data"): voiceDir = "/usr/share/espeak-data"
+elif os.path.exists("/usr/lib/x86_64-linux-gnu/espeak-data"): voiceDir = "/usr/lib/x86_64-linux-gnu/espeak-data"
+else:
+    print ("Content-type: text-plain\n\nUnable to find ESPEAK_DATA_PATH")
+    raise SystemExit
+voiceDir += "/voices"
 
 variants = os.listdir(voiceDir+"/!v")
 if "whisper" in variants and "wisper" in variants: variants.remove("wisper")
@@ -45,16 +69,18 @@ try: speed=int(speed)
 except: speed=defaultSpeed
 if speed<minSpeed or speed>maxSpeed: speed=defaultSpeed
 
-t = f.getfirst("t","")
+t = S(f.getfirst("t",""))
 if len(t)>max_input_size: t=""
 else:
-    try: t.decode('utf-8')
-    except: t="" # not valid utf-8
+    if not type(t)==type(u""):
+        try: t.decode('utf-8')
+        except: t="" # not valid utf-8
     if chr(0) in t: t="" # just in case
 
 if len(t)>stream_if_input_bigger_than:
     # streaming - will need sox to convert
-    if not commands.getoutput("which sox 2>/dev/null"): raise Exception("Cannot find sox")
+    if not getoutput("which sox 2>/dev/null"): raise Exception("Cannot find sox")
+    fname=None
 else:
     # not streaming (so can fill in length etc) - will need a writable file in a private tmp directory, preferably in memory
     worked = 0
@@ -69,15 +95,16 @@ else:
     open(fname2,"w") # raising exception if it's unwritable (try changing to a suitable directory)
 
 # in case espeak can't find a utf-8 locale
-loc=commands.getoutput("locale -a|grep -i 'utf-*8$'|head -1").strip()
+loc=getoutput("locale -a|grep -i 'utf-*8$'|head -1").strip()
 if loc: os.environ["LC_CTYPE"]=loc
 
 def getName(f):
+    f = S(f)
     o=open(f)
     line=""
     for t in range(10):
-        line=o.readline()
-        if "name" in line: return line.split()[1]
+        line=getBuf(o).readline()
+        if u"name".encode('latin1') in line: return S(line.split()[1])
     return f[f.rindex("/")+1:] # assumes it'll be a full pathname
 def isDirectory(directory):
     oldDir = os.getcwd()
@@ -88,10 +115,21 @@ def isDirectory(directory):
     os.chdir(oldDir)
     return ret
 
+def getBuf(f):
+    if hasattr(f,"buffer"): return f.buffer # Python 3
+    else: return f # Python 2
+def doPipe(P,t):
+    if type("")==type(u""): # Python 3
+        P = os.popen(P,"w")
+        if type(t)==type(u""): P.write(t)
+        else: P.buffer.write(t)
+        P.close()
+    else: os.popen(P,"wb").write(t) # Python 2
+
 if t and f.getfirst("qx",""):
     sys.stdout.write("Content-Type: text/plain; charset=utf-8\n\n")
     sys.stdout.flush() # help mathopd
-    os.popen(prog+" -v "+lang+" -q -X -m 2>/dev/null","wb").write(t)
+    doPipe(prog+" -v "+lang+" -q -X -m 2>/dev/null",t)
 elif t:
     prog_with_params = prog+" -v "+lang+"+"+variant+" -s "+str(speed)+" -m"
     # TODO -p 0-99 default 50 (pitch adjustment)
@@ -100,19 +138,20 @@ elif t:
         sys.stdout.write("Content-Type: audio/basic\nContent-Disposition: attachment; filename=\""+t+"_"+lang+".au\"\n\n") # using .au instead of .wav because Windows Media Player doesn't like incorrect length fields in wav.  And make sure it's attachment otherwise Mac OS QuickTime etc can have problems when server is slow
         # problem is, WILL NEED CONVERTING for gradint (unless want to use "sox" on the Windows version before playing via winsound) (but the espeak no-length wav files will probably be wrong on that anyway).  Should be OK because we're doing this only in the case of len(t)>stream_if_input_bigger_than.
         sys.stdout.flush() # help mathopd
-        os.popen(prog_with_params+" --stdout 2>/dev/null | sox -t wav - -t au - 2>/dev/null","wb").write(t)
-    else:
-        os.popen(prog_with_params+" -w "+fname2+" 2>/dev/null","wb").write(t)
+        doPipe(prog_with_params+" --stdout 2>/dev/null | sox -t wav - -t au - 2>/dev/null",t)
+    else: # not streaming
+        doPipe(prog_with_params+" -w "+fname2+" 2>/dev/null",t)
         sys.stdout.write("Content-Type: audio/wav\nContent-Disposition: attachment; filename=\""+t+"_"+lang+".wav\"\n\n")
-        sys.stdout.write(open(fname2,"rb").read())
+        sys.stdout.flush()
+        getBuf(sys.stdout).write(open(fname2,"rb").read())
 else:
-    sys.stdout.write("Content-Type: text/html; charset=utf-8\n\n<HTML><BODY>") # (specify utf-8 here in case accept-charset is not recognised, e.g. some versions of IE6)
-    banner = commands.getoutput(prog+" --help|head -3").strip()
+    sys.stdout.write('Content-Type: text/html; charset=utf-8\n\n<HTML><head><meta name="viewport" content="width=device-width"></head><BODY>') # (specify utf-8 here in case accept-charset is not recognised, e.g. some versions of IE6)
+    banner = S(getoutput(prog+" --help|head -3").strip())
     sys.stdout.write("This is espeak.cgi version "+version+", using <A HREF=http://espeak.sourceforge.net/>eSpeak</A> "+" ".join(banner.split()[1:]))
     if not loc: sys.stdout.write("<br>Warning: could not find a UTF-8 locale; espeak may malfunction on some languages")
-    warnings=commands.getoutput(prog+" -q -x .").strip() # make sure any warnings about locales are output
+    warnings=S(getoutput(prog+" -q -x .").strip()) # make sure any warnings about locales are output
     if warnings: sys.stdout.write("<br>"+warnings)
-    sys.stdout.write("<FORM accept-charset=UTF-8>Text or SSML: <INPUT TYPE=text NAME=t STYLE='width:80%'><br>Language: <SELECT NAME=l>")
+    sys.stdout.write("<FORM method=post accept-charset=UTF-8>Text or SSML: <INPUT TYPE=text NAME=t STYLE='width:80%'><br>Language: <SELECT NAME=l>")
     ld=os.listdir(voiceDir)
     directories = {}
     for f in ld[:]:
@@ -137,9 +176,9 @@ else:
         if v==variant: sys.stdout.write(" SELECTED")
         sys.stdout.write(">"+name+"</OPTION>")
     sys.stdout.write("</SELECT> Speed: <SELECT NAME=s>")
-    for ss in range(minSpeed,maxSpeed,speedStep)+[maxSpeed]:
+    for ss in list(range(minSpeed,maxSpeed,speedStep))+[maxSpeed]:
         sys.stdout.write("<OPTION VALUE="+str(ss))
         if ss==speed: sys.stdout.write(" SELECTED")
         sys.stdout.write(">"+str(ss)+"</OPTION>")
     sys.stdout.write("</SELECT> <INPUT TYPE=submit NAME=qx VALUE=\"View phonemes\"><center><big><INPUT TYPE=submit VALUE=SPEAK></big></center></FORM></BODY></HTML>")
-os.system("rm -rf \""+fname+"\"") # clean up temp dir
+if fname: os.system("rm -rf \""+fname+"\"") # clean up temp dir

server/gradint.cgi

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#  (either Python 2 or Python 3)
+
+program_name = "gradint.cgi v1.38 (c) 2011,2015,2017-25 Silas S. Brown.  GPL v3+"
+
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+
+gradint_dir = "$HOME/gradint" # include samples/prompts
+path_add = "$HOME/gradint/bin" # include sox, lame, espeak, maybe oggenc
+lib_path_add = "$HOME/gradint/lib"
+espeak_data_path = "$HOME/gradint"
+
+import os, os.path, sys, cgi, urllib, time, re
+import tempfile, getpass
+myTmp = tempfile.gettempdir()+os.sep+getpass.getuser()+"-gradint-cgi"
+try: from commands import getoutput # Python 2
+except: from subprocess import getoutput # Python 3
+try: from urllib import quote,quote_plus,unquote # Python 2
+except: from urllib.parse import quote,quote_plus,unquote # Python 3
+try: from importlib import reload # Python 3
+except: pass
+home = os.environ.get("HOME","")
+if not home:
+  try:
+    import pwd
+    home = os.path.expanduser("~{0}".format(pwd.getpwuid(os.getuid())[0]))
+  except: home=0
+  if not home: home = ".." # assume we're in public_html
+gradint_dir = gradint_dir.replace("$HOME",home)
+path_add = path_add.replace("$HOME",home)
+lib_path_add = lib_path_add.replace("$HOME",home)
+espeak_data_path = espeak_data_path.replace("$HOME",home)
+try: import Cookie # Python 2
+except: import http.cookies as Cookie # Python 3
+import random
+if "QUERY_STRING" in os.environ and "&" in os.environ["QUERY_STRING"] and ";" in os.environ["QUERY_STRING"]: os.environ["QUERY_STRING"]=os.environ["QUERY_STRING"].replace(";","%3B") # for dictionary sites to add words that contain semicolon
+try: query = cgi.FieldStorage(encoding="utf-8") # Python 3
+except: query = cgi.FieldStorage() # Python 2
+os.chdir(gradint_dir) ; sys.path.insert(0,os.getcwd())
+os.environ["PATH"] = path_add+":"+os.environ["PATH"]
+if "LD_LIBRARY_PATH" in os.environ: os.environ["LD_LIBRARY_PATH"] = lib_path_add+":"+os.environ["LD_LIBRARY_PATH"]
+else: os.environ["LD_LIBRARY_PATH"] = lib_path_add
+os.environ["ESPEAK_DATA_PATH"] = espeak_data_path
+
+cginame = os.sep+sys.argv[0] ; cginame=cginame[cginame.rindex(os.sep)+1:]
+sys.stderr=open("/dev/null","w") ; sys.argv = []
+gradint = None
+def reinit_gradint(): # if calling again, also redo setup_userID after
+    global gradint,langFullName
+    if gradint:
+      if sys.version_info[0]>2: gradint.map,gradint.filter,gradint.chr=gradint._map,gradint._filter,gradint.unichr # undo Python 3 workaround in preparation for it to be done again, because reload doesn't do this (at least not on all Python versions)
+      gradint = reload(gradint)
+    else: import gradint
+    gradint.waitOnMessage = lambda *args:False
+    langFullName = {}
+    for l in gradint.ESpeakSynth().describe_supported_languages().split():
+        abbr,name = gradint.S(l).split("=")
+        langFullName[abbr]=name.replace("_","-")
+    # Try to work out probable default language:
+    lang = os.environ.get("HTTP_ACCEPT_LANGUAGE","")
+    if lang:
+        for c in [',',';','-']:
+            if c in lang: lang=lang[:lang.index(c)]
+        if not lang in langFullName: lang=""
+    global noGTranslate
+    if lang:
+        gradint.firstLanguage = lang
+        if lang=="en": noGTranslate = True
+        else:
+          gradint.secondLanguage="en" # (most probable default)
+          noGTranslate = lang in gradint.GUI_translations # (unless perhaps any are incomplete)
+    elif " zh-" in os.environ.get("HTTP_USER_AGENT",""): # Chinese iPhone w/out Accept-Language
+      gradint.firstLanguage,gradint.secondLanguage = "zh","en"
+      noGTranslate = True # (don't know if it even pops up on that browser, but anyway)
+
+reinit_gradint()
+
+def main():
+  if "id" in query: # e.g. from redirectHomeKeepCookie
+    queryID = query.getfirst("id")
+    if not re.match("[A-Za-z0-9_.-]",queryID): return htmlOut("Bad query.  Bad, bad query.") # to avoid cluttering the disk if we're being given random queries by an attacker.  IDs we generate are numeric only, but allow alphanumeric in case server admin wants to generate them.  Don't allow =, parens, etc (likely random SQL query)
+    os.environ["HTTP_COOKIE"]="id="+queryID
+    print ('Set-Cookie: id=' + queryID+'; expires=Wed, 1 Dec 2036 23:59:59 GMT') # TODO: S2G
+  if has_userID(): setup_userID() # always, even for justSynth, as it may include a voice selection (TODO consequently being called twice in many circumstances, could make this more efficient)
+  filetype=""
+  if "filetype" in query: filetype=query.getfirst("filetype")
+  if not filetype in ["mp3","ogg","wav"]: filetype="mp3"
+  for k in query.keys():
+    if k.startswith("del-"):
+     k=unquote(unquote(k)) # might be needed
+     if '=' in k:
+       l2,l1 = k[4:].split('=')
+       setup_userID()
+       gradint.delOrReplace(gradint.ensure_unicode(l2),gradint.ensure_unicode(l1),"","","delete")
+       return listVocab(True)
+  if "js" in query: # just synthesize (js=text jsl=language)
+    if "jsl" in query: justSynth(query.getfirst("js"), query.getfirst("jsl"),filetype=filetype)
+    else: justSynth(query.getfirst("js"),filetype=filetype)
+  elif "spk" in query: # speak (l1,l2 the langs, l1w,l2w the words)
+    gradint.justSynthesize="0"
+    if "l2w" in query and query.getfirst("l2w"):
+      gradint.startBrowser=lambda *args:0
+      if query.getfirst("l2")=="zh" and gradint.generalCheck(query.getfirst("l2w"),"zh"): gradint.justSynthesize += "#en Pinyin needs tones.  Please go back and add tone numbers." # speaking it because alert box might not work and we might be being called from HTML5 Audio stuff (TODO maybe duplicate generalCheck in js, if so don't call HTML5 audio, then we can have an on-screen message here)
+      else: gradint.justSynthesize += "#"+query.getfirst("l2").replace("#","").replace('"','')+" "+query.getfirst("l2w").replace("#","").replace('"','')
+    if "l1w" in query and query.getfirst("l1w"): gradint.justSynthesize += "#"+query.getfirst("l1").replace("#","").replace('"','')+" "+query.getfirst("l1w").replace("#","").replace('"','')
+    if gradint.justSynthesize=="0": return htmlOut(withLocalise('You must type a word in the box before pressing the Speak button.')+backLink) # TODO maybe add a Javascript test to the form also, IF can figure out if window.alert works
+    serveAudio(stream = len(gradint.justSynthesize)>100, filetype=filetype)
+  elif "add" in query: # add to vocab (l1,l2 the langs, l1w,l2w the words)
+    if "l2w" in query and query.getfirst("l2w") and "l1w" in query and query.getfirst("l1w"):
+      gradint.startBrowser=lambda *args:0
+      if query.getfirst("l2")=="zh": gcmsg=gradint.generalCheck(query.getfirst("l2w"),"zh")
+      else: gcmsg=None
+      if gcmsg: htmlOut(gradint.B(gcmsg)+gradint.B(backLink))
+      else: addWord(query.getfirst("l1w"),query.getfirst("l2w"),query.getfirst("l1"),query.getfirst("l2"))
+    else: htmlOut(withLocalise('You must type words in both boxes before pressing the Add button.')+backLink) # TODO maybe add a Javascript test to the form also, IF can figure out a way to tell whether window.alert() works or not
+  elif "bulkadd" in query: # bulk adding, from authoring options
+    dirID = setup_userID()
+    def isOK(x):
+      if x[0]=='W':
+        try:
+          int(x[1:])
+          return True
+        except: pass
+    def mycmp(x,y): return cmp(int(x[1:]),int(y[1:]))
+    keyList = sorted(filter(lambda x:isOK(x),query.keys()),mycmp)
+    for k in keyList:
+      l2w,l1w = query.getfirst(k).split('=',1)
+      addWord(l1w,l2w,query.getfirst("l1"),query.getfirst("l2"),False)
+    redirectHomeKeepCookie(dirID,"&dictionary=1") # '1' is special value for JS-only back link; don't try to link to referer as it might be a generated page
+  elif "clang" in query: # change languages (l1,l2)
+    dirID = setup_userID()
+    if (gradint.firstLanguage,gradint.secondLanguage) == (query.getfirst("l1"),query.getfirst("l2")) and not query.getfirst("clang")=="ignore-unchanged": return htmlOut(withLocalise('You must change the settings before pressing the Change Languages button.')+backLink) # (external scripts can set clang=ignore-unchanged)
+    gradint.updateSettingsFile(gradint.settingsFile,{"firstLanguage": query.getfirst("l1"),"secondLanguage":query.getfirst("l2")})
+    redirectHomeKeepCookie(dirID)
+  elif "swaplang" in query: # swap languages
+    dirID = setup_userID()
+    gradint.updateSettingsFile(gradint.settingsFile,{"firstLanguage": gradint.secondLanguage,"secondLanguage":gradint.firstLanguage})
+    redirectHomeKeepCookie(dirID)
+  elif "editsave" in query: # save 'vocab'
+    dirID = setup_userID()
+    if "vocab" in query: vocab=query.getfirst("vocab")
+    else: vocab="" # user blanked it
+    open(gradint.vocabFile,"w").write(vocab)
+    redirectHomeKeepCookie(dirID)
+  elif "edit" in query: # show the edit form
+    dirID = setup_userID()
+    try: v=open(gradint.vocabFile).read()
+    except: v="" # (shouldn't get here unless they hack URLs)
+    htmlOut('<form action="'+cginame+'" method="post"><textarea name="vocab" style="width:100%;height:80%" rows="15" cols="50">'+v+'</textarea><br><input type=submit name=editsave value="Save changes"> | <input type=submit name=placeholder value="Cancel"></form>',"Text edit your vocab list")
+  elif "lesson" in query: # make lesson ("Start lesson" button)
+    setup_userID()
+    gradint.maxNewWords = int(query.getfirst("new")) # (shouldn't need sensible-range check here if got a dropdown; if they really want to hack the URL then ok...)
+    gradint.maxLenOfLesson = int(float(query.getfirst("mins"))*60)
+    # TODO save those settings for next time also?
+    serveAudio(stream = True, inURL = False, filetype=filetype)
+  elif "bigger" in query or "smaller" in query:
+    u = setup_userID() ; global zoom
+    if "bigger" in query: zoom = int(zoom*1.1)
+    else: zoom = int(zoom/1.1 + 0.5)
+    open(u+"-zoom.txt","w").write("%d\n" % zoom)
+    listVocab(True)
+  elif any("variant"+str(c) in query for c in range(max(len(gradint.GUI_translations[v]) for v in gradint.GUI_translations.keys() if v.startswith("@variants-")))):
+    for c in range(max(len(gradint.GUI_translations[v]) for v in gradint.GUI_translations.keys() if v.startswith("@variants-"))): #TODO duplicate code
+      if "variant"+str(c) in query: break
+    u = setup_userID()
+    gradint.updateSettingsFile(u+"-settings.txt",{"scriptVariants":{gradint.GUI_languages.get(gradint.firstLanguage,gradint.firstLanguage):c}})
+    setup_userID() ; listVocab(True)
+  elif "voNormal" in query: # voice option = normal
+    setup_userID()
+    gradint.voiceOption=""
+    gradint.updateSettingsFile(gradint.settingsFile,{"voiceOption":""})
+    listVocab(True)
+  elif "vopt" in query: # set voice option
+    setup_userID()
+    for v in gradint.guiVoiceOptions:
+      if v.lower()=="-"+query.getfirst("vopt").lower():
+        gradint.voiceOption = v
+        gradint.updateSettingsFile(gradint.settingsFile,{"voiceOption":v})
+        break
+    listVocab(True)
+  elif "lFinish" in query:
+    dirID = setup_userID()
+    try: os.rename(gradint.progressFile+'-new',gradint.progressFile)
+    except: pass # probably a duplicate GET
+    try: os.remove(gradint.progressFile+'-ts') # the timestamp file
+    except: pass
+    redirectHomeKeepCookie(dirID)
+  elif not isAuthoringOption(query): listVocab(has_userID()) # default screen
+
+def U(x):
+  try: return x.decode('utf-8')
+  except: return x
+
+def isAuthoringOption(query):
+  # TODO document the ?author=1 option
+  if "author" in query:
+    htmlOut('<form action="'+cginame+'" method="post"><h2>Gradint word list authoring mode</h2>This can help you put word lists on your website. The words will be linked to this Gradint server so your visitors can choose which ones to hear and/or add to their personal lists.<p>Type any text in the box below; use blank lines to separate paragraphs. To embed a word list in your text, type:<br><em>phrase 1</em>=<em>meaning 1</em><br><em>phrase 2</em>=<em>meaning 2</em><br><em>phrase 3</em>=<em>meaning 3</em><br>etc, and <b>make sure there is a blank line before and after the list</b>. Then press <input type=submit name="generate" value="Generate HTML">.<p>Language for phrases: '+langSelect('l2',gradint.secondLanguage)+' and for meanings: '+langSelect('l1',gradint.firstLanguage)+'<p><textarea name="text" style="width:100%;height:80%" rows="15" cols="50"></textarea><br><input type=submit name="generate" value="Generate HTML"></form>',"Word list authoring",links=0)
+    # TODO maybe langSelect for mand+cant together ? (but many wordlists wld be topolect-specific)
+  elif "generate" in query:
+    l1,l2,txt = query.getfirst("l1"),query.getfirst("l2"),query.getfirst("text")
+    paras = "\n".join([l.strip() for l in U(txt).replace("\r\n","\n").replace("\r","\n").split("\n")]).split("\n\n")
+    need_h5a = False
+    for i in xrange(len(paras)):
+        lines = filter(lambda x:x,paras[i].split("\n")) # filter needed for trailing newline on document
+        if allLinesHaveEquals(lines):
+            paras[i] = authorWordList(lines,l1,l2)
+            need_h5a = True
+        # TODO else some wiki markup for links etc ? (but you can alter the HTML after)
+    if need_h5a: h5astr = h5a()
+    else: h5astr = ""
+    htmlOut(HTML_and_preview(h5astr+encodeAmp('<p>'.join(paras))),"HTML result",links=0)
+  else: return False
+  return True
+def allLinesHaveEquals(lines):
+    if not lines: return False
+    for l in lines:
+        if not '=' in l: return False
+    return True
+gradintUrl = os.environ.get("SCRIPT_URI","") # will be http:// or https:// as appropriate
+if not gradintUrl and all(x in os.environ for x in ["REQUEST_SCHEME","SERVER_NAME","SCRIPT_NAME"]): gradintUrl = os.environ["REQUEST_SCHEME"]+"://"+os.environ["SERVER_NAME"]+os.environ["SCRIPT_NAME"]
+if not gradintUrl: gradintUrl = "gradint.cgi" # guessing
+def authorWordList(lines,l1,l2):
+    r=[] ; count = 0
+    # could have target="gradint" in the following, but it may be in a background tab (target="_blank" not recommended as could accumulate many)
+    r.append('<form action="%s" method="post" accept-charset="utf-8"><table style="margin-left:auto;margin-right:auto;border:thin solid blue"><tr><td colspan=3 style="text-align:center"><em>Click on each word for audio</em></td></tr>' % gradintUrl)
+    for l in lines:
+        l2w,l1w = l.split('=',1)
+        r.append('<tr class="notranslate"><td><input type="checkbox" name="W%d" value="%s=%s" checked></td><td>%s</td><td>%s</td></tr>' % (count,l2w,l1w,U(justsynthLink(l2w.encode('utf-8'),l2)).replace('HREF="'+cginame+'?','HREF="'+gradintUrl+'?'),U(justsynthLink(l1w.encode('utf-8'),l1)).replace('HREF="'+cginame+'?','HREF="'+gradintUrl+'?')))
+        count += 1
+    # could have target="gradint" in the following href, but see comment above
+    r.append('<tr><td colspan=3><input type="submit" name="bulkadd" value="Add selected words"> to your <a href="%s">personal list</a></td></tr></table><input type="hidden" name="l1" value="%s"><input type="hidden" name="l2" value="%s"></form>' % (gradintUrl,l1,l2))
+    return ''.join(r)
+def encodeAmp(uniStr):
+  # HTML-ampersand encode when we don't know if the server will be utf-8 after copy/paste
+  r=[]
+  for c in uniStr:
+    if ord(c)>126: r.append("&#"+str(ord(c))+";")
+    else: r.append(c)
+  return ''.join(r)
+def HTML_and_preview(code): return '<h2>HTML code</h2><textarea style="width:100%%;height:40%%" rows=7 cols=50>%s</textarea><h2>Preview</h2>%s' % (code.replace('&','&amp;').replace('<','&lt;'),code)
+
+def justSynth(text,lang="",filetype=""):
+  if lang: lang = lang.replace("#","").replace('"','')+" "
+  gradint.justSynthesize=lang+text.replace("#","").replace('"','')
+  if not filetype in ["mp3","ogg","wav"]: filetype="mp3"
+  serveAudio(stream = len(text)>80, filetype=filetype)
+
+def justsynthLink(text,lang=""): # assumes written function h5a
+  if lang in gradint.synth_partials_voices and gradint.guiVoiceOptions: cacheInfo="&curVopt="+gradint.voiceOption
+  else: cacheInfo=""
+  return '<A HREF="'+cginame+'?js='+gradint.S(quote_plus(text))+'&jsl='+quote_plus(lang)+cacheInfo+'" onClick="return h5a(this);">'+gradint.S(text)+'</A>'
+# TODO if h5a's canPlayType etc works, cld o/p a lesson as a JS web page that does its own 'take out of event stream' and 'progress write-back'.  wld need to code that HERE by inspecting the finished Lesson object, don't call play().
+
+zoom = 100 # in case browser device lacks a zoom UI, we'll provide one
+noGTranslate = False
+def htmlOut(body_u8,title_extra="",links=1):
+    if noGTranslate: print ("Google: notranslate")
+    print ("Content-type: text/html; charset=utf-8\n")
+    if title_extra: title_extra=": "+title_extra
+    print ('<html lang="en"><head><title>Gradint Web edition'+title_extra+'</title>')
+    print ('<meta name="mobileoptimized" content="0"><meta name="viewport" content="width=device-width">')
+    print ('<script>if(window.matchMedia && window.matchMedia("(prefers-color-scheme: dark)").matches)document.write("<style>body,input,textarea { background-color: black; color: #c0c000; } select,input[type=submit],input[type=button] { background-color: #300020; color: #c0c000; } input[type=text] { border: grey groove; } select[disabled],input[disabled] { background-color: #101010; color: #b0b000; } a:link { color: #00b000; } a:visited { color: #00c0c0; } a:hover { color: red; }</style>");</script>')
+    if not zoom==100: print('<style>body{font-size:%d%%;}body *{font-size:100%%;}</style>' % zoom)
+    print ('</head><body>')
+    if type(body_u8)==type(u""): body_u8=body_u8.encode('utf-8')
+    if hasattr(sys.stdout,'buffer'): # Python 3
+      sys.stdout.flush()
+      sys.stdout.buffer.write(body_u8)
+      sys.stdout.flush()
+    else: print(body_u8)
+    print ('<HR>')
+    if links:
+        print ('This is Gradint Web edition.  If you need recorded words or additional functions, please <A HREF="//ssb22.user.srcf.net/gradint/">download the full version of Gradint</A>.')
+        # TODO @ low-priority: Android 3 <input type="file" accept="audio/*;capture=microphone"></input>
+    print ('<p>'+program_name[:program_name.index("(")]+"using "+gradint.program_name[:gradint.program_name.index("(")])
+    print ("</body></html>")
+backLink = ' <A HREF="'+cginame+'" onClick="history.go(-1);return false">Back</A>' # TODO may want to add a random= to the non-js HREF
+
+def serveAudio(stream=0, filetype="mp3", inURL=1):
+  # caller imports gradint (and sets justSynthesize or whatever) first
+  if os.environ.get("HTTP_IF_MODIFIED_SINCE",""):
+    print ("Status: 304 Not Modified\n\n") ; return
+  httpRange = re.match("bytes=([0-9]*)-([0-9]*)$",os.environ.get('HTTP_RANGE','')) # we MUST support Range: for some iOS players (Apple did not follow the HTTP standard of having a sensible fallback if servers respond with 200, and Apache will not do Range for us if we're CGI).  Single Range should be sufficient.
+  if httpRange: httpRange = httpRange.groups()
+  if httpRange==('',''): httpRange = None # must spec one
+  if httpRange:
+    if not httpRange[0]: httpRange=[-int(httpRange[1]),None]
+    elif not httpRange[1]: httpRange=[int(httpRange[0]),None]
+    else: httpRange=[int(httpRange[0]),int(httpRange[1])+1]
+    print ("Status: 206 Partial Content")
+    stream = 0
+  if filetype=="mp3": print ("Content-type: audio/mpeg")
+  else: print ("Content-type: audio/"+filetype) # ok for ogg, wav?
+  if inURL:
+    print ("Last-Modified: Sun, 06 Jul 2008 13:20:05 GMT")
+    print ("Expires: Wed, 1 Dec 2036 23:59:59 GMT") # TODO: S2G
+  print ("Content-disposition: attachment; filename=gradint."+filetype) # helps with some browsers that can't really do streaming
+  gradint.out_type = filetype
+  gradint.waitBeforeStart = 0
+  def mainOrSynth():
+    oldProgress = None ; rollback = False
+    if not gradint.justSynthesize and 'h5a' in query:
+      try: oldProgress = open(gradint.progressFile,'rb').read()
+      except: pass
+      rollback = True
+      if "lesson" in query: random.seed(query.getfirst("lesson")) # so clients that re-GET same lesson from partway through can work
+    try: gradint.main()
+    except SystemExit:
+      if not gradint.justSynthesize:
+        o1,o2 = gradint.write_to_stdout,gradint.outputFile
+        reinit_gradint() ; setup_userID()
+        gradint.write_to_stdout,gradint.outputFile = o1,o2
+        gradint.setSoundCollector(gradint.SoundCollector())
+        gradint.justSynthesize = "en Problem generating the lesson. Check we have prompts for those languages." ; gradint.main()
+        if oldProgress: open(gradint.progressFile,'wb').write(oldProgress)
+        rollback = oldProgress = None
+    if rollback: # roll back pending lFinish
+      os.rename(gradint.progressFile,gradint.progressFile+'-new')
+      if oldProgress: open(gradint.progressFile,'wb').write(oldProgress)
+    # end of def mainOrSynth
+  if stream:
+    print ("")
+    sys.stdout.flush()
+    gradint.write_to_stdout = 1
+    gradint.outputFile="-."+filetype ; gradint.setSoundCollector(gradint.SoundCollector())
+    mainOrSynth()
+  else:
+    gradint.write_to_stdout = 0
+    tempdir = tempfile.mkdtemp()
+    fn,fn2 = tempdir+"/I."+filetype, tempdir+"/O."+filetype
+    if httpRange and "lesson" in query: # try to cache it
+      try: os.mkdir(myTmp)
+      except: pass # exist ok
+      for f in os.listdir(myTmp):
+        if os.stat(myTmp+os.sep+f).st_mtime < time.time()-4000:
+          os.remove(myTmp+os.sep+f)
+      fn = gradint.outputPrefix+str(int(query.getfirst("lesson")))+"."+filetype # (don't be tricked into clobbering paths with non-int lesson IDs)
+    if not os.path.exists(fn):
+      gradint.outputFile=fn
+      gradint.setSoundCollector(gradint.SoundCollector())
+      mainOrSynth()
+    if httpRange:
+      total = os.stat(fn).st_size
+      open(fn2,"wb").write(open(fn,"rb").read()[httpRange[0]:httpRange[1]])
+      if httpRange[0]<0: httpRange[0] += total
+      if not httpRange[1]: httpRange[1] = total
+      print("Content-Range: bytes %d-%d/%d" % (httpRange[0],httpRange[1]-1,total))
+    else: fn2 = fn
+    print ("Content-Length: "+repr(os.stat(fn2).st_size)+"\n")
+    sys.stdout.flush()
+    os.system("cat "+fn2) # components already validated so no quoting required
+    os.system("rm -r "+tempdir)
+
+def addWord(l1w,l2w,l1,l2,out=True):
+    if out: dirID=setup_userID()
+    if not (gradint.firstLanguage,gradint.secondLanguage) == (l1,l2):
+      if not ((gradint.firstLanguage,gradint.secondLanguage) == (l2,l1) and "HTTP_REFERER" in os.environ and not cginame in os.environ["HTTP_REFERER"]): gradint.updateSettingsFile(gradint.settingsFile,{"firstLanguage": l1,"secondLanguage":l2})
+      gradint.firstLanguage,gradint.secondLanguage = l1,l2
+    if (l1w+"_"+l1,l2w+"_"+l2) in map(lambda x:x[1:],gradint.parseSynthVocab(gradint.vocabFile,forGUI=1)):
+      if out: htmlOut(withLocalise('This word is already in your list.')+backLink)
+      return
+    gradint.appendVocabFileInRightLanguages().write(gradint.B(l2w)+gradint.B("=")+gradint.B(l1w)+gradint.B("\n"))
+    if not out: return
+    if "HTTP_REFERER" in os.environ and not cginame in os.environ["HTTP_REFERER"]: extra="&dictionary="+quote(os.environ["HTTP_REFERER"])
+    else: extra=""
+    redirectHomeKeepCookie(dirID,extra)
+
+def redirectHomeKeepCookie(dirID,extra=""):
+    dirID = gradint.S(dirID) # just in case
+    print ("Location: "+cginame+"?random="+str(random.random())[2:]+"&id="+dirID[dirID.rindex("/")+1:]+extra+"\n")
+
+def langSelect(name,curLang):
+    curLang = gradint.espeak_language_aliases.get(curLang,curLang)
+    return '<select name="'+name+'">'+''.join(['<option value="'+abbr+'"'+gradint.cond(abbr==curLang," selected","")+'>'+localise(abbr)+' ('+abbr+')'+'</option>' for abbr in sorted(langFullName.keys())])+'</select>'
+
+def numSelect(name,nums,curNum): return '<select name="'+name+'">'+''.join(['<option value="'+str(num)+'"'+gradint.cond(num==curNum," selected","")+'>'+str(num)+'</option>' for num in nums])+'</select>'
+
+def localise(x,span=0):
+    r=gradint.localise(x)
+    if r==x: return langFullName.get(gradint.espeak_language_aliases.get(x,x),x)
+    if span==1: r="<span lang=\""+gradint.firstLanguage+"\">"+r+"</span>"
+    elif span==2: r+='" lang="'+gradint.firstLanguage
+    if type(r)==type("")==type(u""): return r # Python 3
+    else: return r.encode('utf-8') # Python 2
+for k,v in {"Swap":{"zh":u"交换","zh2":u"交換"},
+            "Text edit":{"zh":u"文本编辑"},
+            "Delete":{"zh":u"删除","zh2":u"刪除"},
+            "Really delete this word?":{"zh":u"真的删除这个词?","zh2":u"真的刪除這個詞?"},
+            "Your word list":{"zh":u"你的词汇表","zh2":u"你的詞彙表"},
+            "click for audio":{"zh":u"击某词就听声音","zh2":u"擊某詞就聽聲音"},
+            "Repeats":{"zh":u"重复计数","zh2":u"重複計數"},
+            "To edit this list on another computer, type":{"zh":u"要是想在其他的电脑或手机编辑这个词汇表，请在别的设备打","zh2":u"要是想在其他的電腦或手機編輯這個詞彙表，請在別的設備打"},
+            "Please wait while the lesson starts to play":{"zh":u"稍等本课正开始播放","zh2":u"稍等本課正開始播放"},
+            "Bigger":{"zh":u"大"},"Smaller":{"zh":u"小"},
+            'You must type a word in the box before pressing the Speak button.':{"zh":u"按‘发音’前，应该框里打字。","zh2":u"按‘發音’前，應該框裡打字。"},
+            'You must type words in both boxes before pressing the Add button.':{"zh":u"按‘添加’前，应该在两框里打字。","zh2":u"按‘添加’前，應該在兩框裡打字。"},
+            'You must change the settings before pressing the Change Languages button.':{"zh":u"按‘选择其他语言’前，应该转换语言设定。","zh2":u"按‘選擇其他語言’前，應該轉換語言設定。"},
+            'This word is already in your list.':{"zh":u"本词已经在您的词汇表。","zh2":u"本詞已經在您的詞彙表。"},
+            "Your word list is empty.":{"zh":u"词汇表没有词汇，加一些吧","zh2":u"詞彙表沒有詞彙，加一些吧"}
+            }.items():
+  if not k in gradint.GUI_translations: gradint.GUI_translations[k]=v
+
+def withLocalise(x): return x+"  "+localise(x,1)
+
+def h5a():
+    body = """<script><!--
+function h5a(link,endFunc) { if (document.createElement) {
+   var ae = document.createElement('audio');
+   function cp(t,lAdd) { if(ae.canPlayType && function(s){return s!="" && s!="no"}(ae.canPlayType(t))) {
+       if (link.href) ae.setAttribute('src', link.href+lAdd);
+       else ae.setAttribute('src', link+lAdd);
+       if (typeof endFunc !== 'undefined') { ae.addEventListener("ended", endFunc, false); ae.addEventListener("timeupdate",function(e){t=ae.currentTime;m=Math.floor(t/60);t=Math.floor(t%60);document.forms[0].lesson.value=m+(t<10?":0":":")+t},false) }
+       ae.play(); return true; // Safari can say "Unhandled Promise Rejection: AbortError: The operation was aborted." in console log, but plays anyway when loaded
+    } return false; }
+   if (cp('audio/mpeg','')) return false;"""
+    if gradint.got_program("oggenc"): body += """else if (cp('audio/ogg',"&filetype=ogg")) return false;"""
+    body += """} return true; }
+//--></script>"""
+    return body
+def hasVoiceOptions(l):
+    if not l in gradint.synth_partials_voices: return False
+    if not gradint.guiVoiceOptions: return False
+    try: voices = os.listdir(gradint.partialsDirectory+os.sep+l)
+    except: voices = []
+    for v in voices:
+        if "-" in v and v[:v.index("-")] in voices: return True
+def listVocab(hasList): # main screen
+    firstLanguage,secondLanguage = gradint.firstLanguage, gradint.secondLanguage
+    # TODO button onClick: careful of zh w/out tones, wld need to JS this
+    body = h5a() + '<center><form action="'+cginame+'">'
+    body += '<input type=submit name=bigger value="%s"> | <input type=submit name=smaller value="%s">' % (localise("Bigger"),localise("Smaller"))
+    GUIlang = gradint.GUI_languages.get(firstLanguage,firstLanguage)
+    if "@variants-"+GUIlang in gradint.GUI_translations:
+      body += " -- " # separating from big/small
+      count = 0
+      for variant in gradint.GUI_translations["@variants-"+GUIlang]:
+        if count: body += " | "
+        body += '<input type=submit name="variant'+str(count)+'" value="'+gradint.cond(type("")==type(u""),variant,variant.encode('utf-8'))+'"'+gradint.cond(gradint.scriptVariants.get(GUIlang,0)==count,' disabled="disabled"',"")+'>'
+        count += 1
+    body += "<br>"
+    gotVoiceOptions = (hasVoiceOptions(gradint.secondLanguage) or hasVoiceOptions(gradint.firstLanguage))
+    if gotVoiceOptions:
+      body += 'Voice option: <input type=submit name=voNormal value="Normal"'+gradint.cond(gradint.voiceOption=="",' disabled="disabled"',"")+'>'
+      for v in gradint.guiVoiceOptions: body += ' | <input type=submit name=vopt value="'+v[1].upper()+v[2:]+'"'+gradint.cond(gradint.voiceOption==v,' disabled="disabled"',"")+'>'
+      body += '<input type=hidden name=curVopt value="'+gradint.voiceOption+'">' # ignored by gradint.cgi but needed by browser cache to ensuer 'change voice option and press Speak again' works
+      body += '<br>'
+    # must have autocomplete=off if capturing keycode 13
+    if gotVoiceOptions: cacheInfo="&curVopt="+gradint.voiceOption
+    else: cacheInfo=""
+    body += (localise("Word in %s",1) % localise(secondLanguage))+': <input type=text name=l2w autocomplete=off onkeydown="if(event.keyCode==13) {document.forms[0].spk.click();return false} else return true" onfocus="document.forms[0].onsubmit=\'document.forms[0].onsubmit=&quot;return true&quot;;document.forms[0].spk.click();return false\'" onblur="document.forms[0].onsubmit=\'return true\'"> <input type=submit name=spk value="'+localise("Speak",2)+'" onClick="if (!document.forms[0].l1w.value && !document.forms[0].l2w.value) return true; else return h5a(\''+cginame+'?spk=1&l1w=\'+document.forms[0].l1w.value+\'&l2w=\'+document.forms[0].l2w.value+\'&l1=\'+document.forms[0].l1.value+\'&l2=\'+document.forms[0].l2.value+\''+cacheInfo+'\');"><br>'+(localise("Meaning in %s",1) % localise(firstLanguage))+': <input type=text name=l1w autocomplete=off onkeydown="if(event.keyCode==13) {document.forms[0].add.click();return false} else return true" onfocus="document.forms[0].onsubmit=\'document.forms[0].onsubmit=&quot;return true&quot;;document.forms[0].add.click();return false\'" onblur="document.forms[0].onsubmit=\'return true\'"> <input type=submit name=add value="'+(localise("Add to %s",2) % localise("vocab.txt").replace(".txt",""))+'"><script><!--\nvar emptyString="";document.write(\' <input type=submit name=placeholder value="'+localise("Clear input boxes",2)+'" onClick="document.forms[0].l1w.value=document.forms[0].l2w.value=emptyString;document.forms[0].l2w.focus();return false">\')\n//--></script><p><nobr>'+localise("Your first language",1)+': '+langSelect('l1',firstLanguage)+'</nobr> <nobr>'+localise("second",1)+': '+langSelect('l2',secondLanguage)+'</nobr> <nobr><input type=submit name=clang value="'+localise("Change languages",2)+'"><input type=submit name=swaplang value="'+localise("Swap",2)+'"></nobr>' # onfocus..onblur updating onsubmit is needed for iOS "Go" button
+    def htmlize(l,lang):
+       if type(l)==type([]) or type(l)==type(()): return htmlize(l[-1],lang)
+       l = gradint.B(l)
+       if gradint.B("!synth:") in l: return htmlize(l[l.index(gradint.B("!synth:"))+7:l.rfind(gradint.B("_"))],lang)
+       return justsynthLink(l,lang)
+    def deleteLink(l1,l2):
+       r = []
+       for l in [l2,l1]:
+         if type(l)==type([]) or type(l)==type(()) or not gradint.B("!synth:") in gradint.B(l): return "" # Web-GUI delete in poetry etc not yet supported
+         l = gradint.B(l)
+         r.append(gradint.S(quote(l[l.index(gradint.B("!synth:"))+7:l.rfind(gradint.B("_"))])))
+       r.append(localise("Delete",2))
+       return ('<td><input type=submit name="del-%s%%3d%s" value="%s" onClick="return confirm(\''+localise("Really delete this word?")+'\');"></td>') % tuple(r)
+    if hasList:
+       gradint.availablePrompts = gradint.AvailablePrompts() # needed before ProgressDatabase()
+       # gradint.cache_maintenance_mode=1 # don't transliterate on scan -> NO, including this scans promptsDirectory!
+       gradint.ESpeakSynth.update_translit_cache=lambda *args:0 # do it this way instead
+       data = gradint.ProgressDatabase().data ; data.reverse()
+       if data: hasList = "<p><table style=\"border: thin solid green\"><caption><nobr>"+localise("Your word list",1)+"</nobr> <nobr>("+localise("click for audio",1)+")</nobr> <input type=submit name=edit value=\""+localise("Text edit",2)+"\"></caption><tr><th>"+localise("Repeats",1)+"</th><th>"+localise(gradint.secondLanguage,1)+"</th><th>"+localise(gradint.firstLanguage,1)+"</th></tr>"+"".join(["<tr class=\"notranslate\"><td>%d</td><td lang=\"%s\">%s</td><td lang=\"%s\">%s</td>%s" % (num,gradint.secondLanguage,htmlize(dest,gradint.secondLanguage),gradint.firstLanguage,htmlize(src,gradint.firstLanguage),deleteLink(src,dest)) for num,src,dest in data])+"</table>"
+       else: hasList=""
+    else: hasList=""
+    if hasList: body += '<p><table style="border:thin solid blue"><tr><td>'+numSelect('new',range(2,10),gradint.maxNewWords)+' '+localise("new words in")+' '+numSelect('mins',[15,20,25,30],int(gradint.maxLenOfLesson/60))+' '+localise('mins')+""" <input type=submit name=lesson value="""+'"'+localise("Start lesson",2)+"""" onClick="document.forms[0].lesson.disabled=1; document.forms[0].lesson.value=&quot;"""+localise("Please wait while the lesson starts to play")+"""&quot;;document.d0=new Date();return h5a('"""+cginame+'?lesson='+str(random.random())[2:]+"""&h5a=1&new='+document.forms[0].new.value+'&mins='+document.forms[0].mins.value,function(){if(new Date()-document.d0>60000)location.href='"""+cginame+'?lFinish='+str(random.random())[2:]+"""';else{document.forms[0].lesson.value='PLAY ERROR'}})"></td></tr></table>""" # when lesson ended, refresh with lFinish which saves progress (interrupts before then cancel it), but don't save progress if somehow got the ended event in 1st minute as that could be a browser issue
+    if "dictionary" in query:
+        if query.getfirst("dictionary")=="1": body += '<script><!--\ndocument.write(\'<p><a href="javascript:history.go(-1)">'+localise("Back to referring site",1)+'</a>\')\n//--></script>' # apparently it is -1, not -2; the redirect doesn't count as one (TODO are there any JS browsers that do count it as 2?)
+        else: body += '<p><a href="'+query.getfirst("dictionary")+'">'+localise("Back to dictionary",1)+'</a>' # TODO check for cross-site scripting
+    if hasList:
+      if "://" in gradintUrl: hasList += "<p>"+localise("To edit this list on another computer, type",1)+" <kbd>"+gradintUrl.replace(".","<wbr>.").replace("/","<wbr>/")+"?id="+re.sub("([0-9]{4})(?!$)",r"\1<wbr><span><!-- (this is not a phone number) --></span>",getCookieId())+"</kbd>" # span needed for iOS at least
+    else: hasList="<p>"+localise("Your word list is empty.",1)
+    body += hasList
+    htmlOut(body+'</form></center><script><!--\ndocument.forms[0].l2w.focus()\n//--></script>')
+
+def has_userID(): # TODO: can just call getCookieId with not too much extra overhead
+    cookie_string = os.environ.get('HTTP_COOKIE',"")
+    if cookie_string:
+        cookie = Cookie.SimpleCookie()
+        cookie.load(cookie_string)
+        return 'id' in cookie
+
+def getCookieId():
+    cookie_string = os.environ.get('HTTP_COOKIE',"")
+    if not cookie_string: return
+    cookie = Cookie.SimpleCookie()
+    cookie.load(cookie_string)
+    if 'id' in cookie: return cookie['id'].value.replace('"','').replace("'","").replace("\\","")
+
+def setup_userID():
+    # MUST call before outputting headers (may set cookie)
+    # Use the return value of this with -settings.txt, -vocab.txt etc
+    if cginame=="gradint.cgi": dirName = "cgi-gradint-users" # as previous versions
+    else: dirName = cginame+"-users" # TODO document this feature (you can symlink something-else.cgi to gradint.cgi and it will have a separate user directory) (however it still reports gradint.cgi on the footer)
+    if not os.path.exists(dirName): os.system("mkdir "+dirName)
+    userID = getCookieId()
+    need_write = (userID and not os.path.exists(dirName+'/'+userID+'-settings.txt')) # maybe it got cleaned up
+    if not userID:
+        while True:
+            userID = str(random.random())[2:]
+            if not os.path.exists(dirName+'/'+userID+'-settings.txt'): break
+        open(dirName+'/'+userID+'-settings.txt','w') # TODO this could still be a race condition (but should be OK under normal circumstances)
+        need_write = 1
+        print ('Set-Cookie: id=' + userID+'; expires=Wed, 1 Dec 2036 23:59:59 GMT') # TODO: S2G
+    userID0, userID = userID, dirName+os.sep+userID # already validated
+    gradint.progressFileBackup=gradint.pickledProgressFile=None
+    gradint.vocabFile = userID+"-vocab.txt"
+    gradint.progressFile = userID+"-progress.txt"
+    gradint.settingsFile = userID+"-settings.txt"
+    gradint.outputPrefix = myTmp+os.sep+userID0+"-"
+    if need_write: gradint.updateSettingsFile(gradint.settingsFile,{'firstLanguage':gradint.firstLanguage,'secondLanguage':gradint.secondLanguage})
+    else: gradint.readSettings(gradint.settingsFile)
+    gradint.auto_advancedPrompt=1 # prompt in L2 if we don't have L1 prompts on the server, what else can we do...
+    if os.path.exists(userID+"-zoom.txt"):
+      global zoom ; zoom = int(open(userID+"-zoom.txt").read().strip())
+    return userID
+
+try: main()
+except Exception as e:
+  print ("Content-type: text/plain; charset=utf-8\n")
+  sys.stdout.flush()
+  import traceback
+  try: traceback.print_exc(file=sys.stdout)
+  except: pass
+  sys.stdout.flush()
+  if hasattr(sys.stdout,"buffer"): buf = sys.stdout.buffer
+  else: buf = sys.stdout
+  buf.write(repr(e).encode("utf-8"))

server/lesson-table.py

+#!/usr/bin/env python
+# (compatible with both Python 2 and Python 3)
+
+# Script to generate an HTML table of the contents of a lesson
+# for summarizing it to a teacher or native speaker.
+# Reads from progressFile and progressFileBackup.
+
+# Version 1.07 (c) 2011, 2020-21, 2025 Silas S. Brown.  License: GPL
+
+# Example use:
+# export samples_url=http://example.org/path/to/samples/ # or omit
+# python lesson-table.py [gradint-params] | ssh some-server 'mutt -e "set record = \"\";" -e "set charset=\"utf-8\"; set send_charset=\"utf-8\"; set content_type=\"text/html\";" to-address -s "Gradint report"' || echo Send failed
+
+import gradint, os
+samples_url = os.getenv("samples_url","")
+
+from gradint import B,S
+newpf = gradint.progressFile
+gradint.progressFile = gradint.progressFileBackup
+gradint.pickledProgressFile=None
+
+mergeIn = gradint.scanSamples()+gradint.parseSynthVocab(gradint.vocabFile)
+
+oldProg = gradint.ProgressDatabase(alsoScan=0)
+oldProg.data += oldProg.unavail # because it might be available in newProg
+gradint.mergeProgress(oldProg.data,mergeIn)
+opd = {}
+for tries,l1,l2 in oldProg.data:
+  key = gradint.norm_filelist(l1,l2)
+  if tries: opd[key]=tries
+del oldProg
+gradint.progressFile = newpf
+newProg = gradint.ProgressDatabase(alsoScan=0)
+gradint.mergeProgress(newProg.data,mergeIn)
+del mergeIn
+changes = [] ; count=0
+gradint.sort(newProg.data,gradint.cmpfunc)
+for tries,l1,l2 in newProg.data:
+  if not tries: continue
+  key = gradint.norm_filelist(l1,l2)
+  oldTries = opd.get(key,0)
+  if not oldTries==tries: changes.append((oldTries,count,tries-oldTries,S(l1),S(l2)))
+  count += 1
+del newProg,opd
+changes.sort()
+print ('<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"><title>Gradint lesson report</title><meta name="mobileoptimized" content="0"><meta name="viewport" content="width=device-width"><script>if(window.matchMedia && window.matchMedia("(prefers-color-scheme: dark)").matches)document.write("<style>body { background-color: black; color: #c0c000; } a:link { color: #00b000; } a:visited { color: #00c0c0; } a:hover { color: red; }</style>");if(navigator.languages && navigator.languages.indexOf("en")>=0) document.write(\'<meta name="google" content="notranslate">\')</script></head><body><h2>Gradint lesson report</h2>')
+if gradint.unix and gradint.got_program("zgrep"):
+  print (os.popen("zgrep '^# collection=' \"%s\"" % gradint.progressFile).read()[2:].rstrip())
+print ('<table border><tr><th>Repeats before</th><th>Repeats today</th><th>Question</th><th>Answer</th></tr>') # (have Question/Answer order rather than Word/Meaning, because if it's L2-only poetry then the question is the previous line, which is not exactly "meaning")
+  
+had_h5a = False
+def h5aCode(filename):
+  r = real_h5aCode(filename)
+  if r:
+    global had_h5a
+    if not had_h5a:
+      had_h5a = True
+      print ("""<script language="Javascript"><!--
+function h5a(link,type) { if (document.createElement) {
+   var ae = document.createElement('audio');
+   if (ae.canPlayType && function(s){return s!="" && s!="no"}(ae.canPlayType(type))) {
+     if (link.href) ae.setAttribute('src', link.href);
+     else ae.setAttribute('src', link);
+     ae.play();
+     return false; } } return true; }
+//--></script>""")
+  return r
+def real_h5aCode(filename):
+  if filename.endswith(gradint.dotmp3): return ' onClick="javascript:return h5a(this,\'audio/mpeg\')"'
+  elif filename.endswith(gradint.dotwav): return ' onClick="javascript:return h5a(this,\'audio/wav\')"'
+  else: return ""
+
+def wrappable(f):
+  z = u'\u200b' # zero-width space
+  if not type(u"")==type(""): z=z.encode('utf-8') # Py2
+  return f.replace(os.sep,os.sep+z).replace('_',z+'_')
+def checkVariant(l,ensureTxt=0):
+  l=S(l)
+  if os.sep in l: fname=l[l.rindex(os.sep)+1:]
+  else: fname=l
+  variants = map(S,gradint.variantFiles.get(B(gradint.samplesDirectory+os.sep+l),[fname]))
+  if fname in variants: return l # ok
+  # else no default variant, need to pick one for the link
+  for v in variants:
+    if ensureTxt:
+      if not v.endswith(gradint.dottxt): continue
+    elif v.endswith(gradint.dottxt): continue
+    if not os.sep in l: return v
+    return l[:l.rindex(os.sep)+1]+v
+def link(l):
+  if type(l)==type([]): return link(l[-1])
+  l = S(l)
+  if l.lower().endswith(gradint.dottxt): l="!synth:"+S(gradint.u8strip(gradint.read(gradint.samplesDirectory+os.sep+checkVariant(l,1)))).strip(gradint.wsp)+"_"+gradint.languageof(l)
+  if "!synth:" in l:
+    if gradint.languageof(l) not in [gradint.firstLanguage,gradint.secondLanguage]: l=S(gradint.textof(l))+" ("+gradint.languageof(l)+")"
+    else: l=S(gradint.textof(l))
+    return l.replace('&','&amp;').replace('<','&lt;')
+  if samples_url: return '<A HREF="'+samples_url+checkVariant(l)+'"'+h5aCode(checkVariant(l))+'>'+wrappable(l)+'</A>'
+  return wrappable(l).replace('&','&amp;').replace('<','&lt;')
+for b4,pos,today,l1,l2 in changes: print ('<tr><td>%d</td><td>%d</td><td class="notranslate">%s</td><td class="notranslate">%s</td></tr>' % (b4,today,link(l1),link(l2)))
+print ('</table></body></html>')

gradint-build/samples/utils/safety-check-progressfile.py → server/safety-check-progressfile.py

+#!/usr/bin/env python
+# (either Python 2 or Python 3)
+
 # safety-check-progressfile.py:
 # The purpose of this script is to check
 # progress.txt for safety.  Because it's

server/samples.cgi

+#!/bin/bash
+
+# Gradint online samples browser v1.1 (c) 2011,2013 Silas S. Brown.  License: GPL
+
+# Works as an "indexing" CGI.
+# To set up in Apache, make .htaccess with:
+# Options -Indexes
+# ErrorDocument 403 /~your-user-ID/cgi-bin/samples.cgi
+# <FilesMatch "\.(txt)$">
+# ForceType 'text/plain; charset=UTF-8' 
+# </FilesMatch>
+
+# and change the /home/ssb22 in the script below.
+
+# To set up in mathopd, configure like this:
+#		Control {
+#		  Alias /samples
+#		  Location /home/userID/gradint/samples/
+#		  AutoIndexCommand /home/userID/path/to/samples.cgi
+#		}
+
+# and delete the REQUEST_URI logic below.
+
+# You can override this script in selected (sub)directories
+# by making index.html files for those.
+
+if ! test "a$REQUEST_URI" == a; then
+  cd "/home/ssb22/public_html/$(echo "$REQUEST_URI"|sed -e 's/?.*//')"
+fi # else assume the server put us in the right directory, like mathopd does
+
+if echo "$SERVER_SOFTWARE"|grep Apache >/dev/null; then
+  echo "Status: 200 OK" # overriding the 403
+fi # (mathopd doesn't need this, and not tested with all mathopd versions)
+
+Filename="$(pwd|sed -e 's,.*/,,').zip"
+
+if test "$QUERY_STRING" == zip || test "a$(echo "$REQUEST_URI"|sed -e 's/.*?//')" == azip; then
+  echo Content-type: application/zip
+  echo "Content-Disposition: attachment; filename=$Filename"
+  echo
+  cd .. ; zip -9r - "$(echo "$Filename"|sed -e s/.zip$//)"
+else
+  echo "Content-type: text/html; charset=utf-8"
+  echo
+  echo "<HTML><BODY><A HREF=\"..\">Parent directory</A> |"
+  echo "<A HREF=\"./?zip\">Download $Filename</A> (expands to $(du -h --apparent-size -s|cut -f1))"
+  echo "<h2>Contents of $Filename</h2><UL>"
+  cat <<EOF
+<script language="Javascript"><!--
+function h5a(link) {
+ if (!link.nextSibling) return true;
+ if (link.nextSibling.src) {
+   link.nextSibling.play();
+   return false;
+ } else {
+   var ae = document.createElement('audio');
+   var atype;
+   if (link.href.match("mp3$")) atype="audio/mpeg";
+   else if (link.href.match("wav$")) atype="audio/wav";
+   else if (link.href.match("ogg$")) atype="audio/ogg";
+   else return true;
+   if (ae.canPlayType && function(s){return s!="" && s!="no"}(ae.canPlayType(atype))) {
+ ae.setAttribute('src', link.href);
+ ae.setAttribute('controls', 'controls');
+ link.parentNode.insertBefore(ae,link.nextSibling);
+ ae.play();
+ return false;
+   }
+} return true;}
+//--></script>
+EOF
+  for N in *; do
+    Size=$(du -h --apparent-size -s "$N"|cut -f1)
+    if echo "$N"|grep '\.txt$'>/dev/null && echo $Size|grep '^[0-9]*$' >/dev/null;then Size="$(cat "$N")";else Size="($Size)"; fi
+    echo "<LI><A HREF=\"$N\" onClick=\"javascript:return h5a(this)\">$N</A> $Size</LI>"
+  done
+  echo "</UL></BODY></HTML>"
+fi

gradint-build/samples/utils/vocab2html.py → server/vocab2html.py

@@ -13,34 +13,35 @@
 # be used.  E.g.: export ESPEAK_CGI_URL="/~userID/espeak.cgi"
 # (TODO: this script ignores the possibility of synthesizing phrases from partials)
 
-# (c) Silas S. Brown, License: GPL
+# Version 1.2, (c) Silas S. Brown, License: GPL
 
 from gradint import *
 if not synthCache: synthCache_contents = []
 langs=[secondLanguage,firstLanguage]
 o=open(vocabFile,"rU")
 justHadP=1
-sys.stdout.write("<HEAD><META HTTP_EQUIV=Content-type CONTENT=\"text/html; charset=utf-8\"></HEAD>\n") # (assume utf8 in case there's any hanzi in lily, or in espeak cantonese voice or whatever - but TODO what if using another charset for another language?)
-for l in o.readlines():
+print ('<html><HEAD><META HTTP-EQUIV=Content-type CONTENT="text/html; charset=utf-8"><meta name="viewport" content="width=device-width"></HEAD><body>') # (assume utf8 in case there's any hanzi, but TODO what if using another charset for another language?)
+for l in o:
   l2=l.lower()
   if l2.startswith("set language ") or l2.startswith("set languages "): langs=l.split()[2:]
   if not l.strip():
     # blank line
-    if not justHadP: sys.stdout.write("<P>")
+    if not justHadP: print ("<P>")
     justHadP=1 ; continue
-  if not justHadP: sys.stdout.write("<BR>")
+  if not justHadP: print ("<BR>")
   if l2.startswith("set language ") or l2.startswith("set languages ") or l2.startswith("limit on") or l2.startswith("limit off") or l2.startswith("begin poetry") or l2.startswith("end poetry"):
-    sys.stdout.write("<EM>%s</EM>" % (l,))
+    print ("<EM>%s</EM>" % (l,))
   elif l2.startswith("#"):
     # comment (and may be part of multi-line comment)
-    if not l[1:].strip().startswith("<!--"): sys.stdout.write("<small>#</small> ")
-    sys.stdout.write(l[1:])
+    if not l[1:].strip().startswith("<!--"): print ("<small>#</small> ")
+    print (l[1:])
   else:
     # vocab line
     langsAndWords=zip(langs,map(lambda x:x.strip(),l.split("=")))
     out = []
     for lang,word in langsAndWords:
-      fname=synthCache_transtbl.get(word.lower()+"_"+lang+dotwav,word.lower()+"_"+lang+dotwav)
+      lang,word = S(lang),S(word)
+      fname=S(synthCache_transtbl.get(word.lower()+"_"+lang+dotwav,word.lower()+"_"+lang+dotwav))
       found = 0
       for fn2 in [fname,fname.replace(dotwav,dotmp3)]:
           if fn2 in synthCache_contents:
@@ -48,8 +49,10 @@ for l in o.readlines():
               found = 1 ; break
       if not found:
           if os.getenv("ESPEAK_CGI_URL"):
-              import urllib
-              out.append("<A HREF=\""+os.getenv("ESPEAK_CGI_URL")+"?"+urllib.urlencode({"t":word,"l":lang})+"\">"+word+"</A>")
+              try: from urllib import urlencode # Python 2
+              except: from urllib.parse import urlencode # Python 3
+              out.append("<A HREF=\""+os.getenv("ESPEAK_CGI_URL")+"?"+urlencode({"t":word,"l":lang})+"\">"+word+"</A>")
           else: out.append(word)
-    sys.stdout.write(" = ".join(out))
+    print (" = ".join(out))
   justHadP=0
+print ("</body></html>")

src/Makefile

+../gradint.py:
+	make -C .. gradint.py

gradint-build/src/booktime.py → src/booktime.py

-# This file is part of the source code of
-# gradint v0.9962 (c) 2002-2010 Silas S. Brown. GPL v3+.
+# This file is part of the source code of Gradint
+# (c) Silas S. Brown.
 #    This program is free software; you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by
 #    the Free Software Foundation; either version 3 of the License, or
@@ -23,17 +23,14 @@
 
 def initialGlue(): return Glue(0,maxLenOfLesson)
 
-try: import bisect
+try: from bisect import insort
 except:
-    class bisect: pass
-    bisect=bisect()
     def insort(l,item):
         l.append(item) ; l.sort()
-    bisect.insort = insort
 class Schedule(object):
     # A sorted list of (start,finish) times that are booked
     def __init__(self): self.bookedList = []
-    def book(self,start,finish): bisect.insort(self.bookedList,(start,finish))
+    def book(self,start,finish): insort(self.bookedList,(start,finish))
 
 earliestAllowedEvent = 0 # for "don't start before" hacks, so can keep all initial glue starting at 0
 
@@ -85,6 +82,7 @@ class GlueOrEvent(object):
         # i.e. to the events that matter to addToEvents.
         # (and only if they don't already have such an attribute, so we can put the exceptions first.)
         if not hasattr(self,name): exec('self.'+name+'='+repr(value))
+    def setOnLastLeaf(self,name,value): self.setOnLeaves(name,value)
 class Event (GlueOrEvent):
     def __init__(self,length):
         GlueOrEvent.__init__(self,length)
@@ -92,7 +90,10 @@ class Event (GlueOrEvent):
         # Playing a "silence event".  Normally don't need to do anything.
         # However, if playing in real time and the scheduler is behind, then
         # this delay is probably important - so have at least most of it.
-        if not soundCollector: mysleep(min(3,self.length*0.7))
+        # AND if not real-time then we DON'T want to beep during this silence
+        # (long-phrase/pause/long-answer is different from pause between sequences)
+        if soundCollector: soundCollector.addSilence(self.length*0.7,False)
+        else: mysleep(min(3,self.length*0.7))
 class CompositeEvent (Event):
     # An event made up of several others in sequence with
     # nothing intervening
@@ -109,6 +110,7 @@ class CompositeEvent (Event):
         for e in self.eventList: e.play()
     def setOnLeaves(self,name,value):
         for e in self.eventList: e.setOnLeaves(name,value)
+    def setOnLastLeaf(self,name,value): self.eventList[-1].setOnLastLeaf(name,value)
     def makesSenseToLog(self):
         if hasattr(self,"is_prompt"): return not self.is_prompt
         for e in self.eventList:
@@ -118,10 +120,7 @@ class Glue (GlueOrEvent):
     def __init__(self,length,plusMinus):
         GlueOrEvent.__init__(self,length,plusMinus,1)
 
-def sgn(a):
-    # Not all versions of Python have this built-in
-    if a: return a/abs(a)
-    else: return 1
+def sgn(a): return [1,-1][a<0]
 
 class StretchedTooFar(Exception): pass
 class GluedEvent(object):
@@ -164,6 +163,7 @@ class GluedEvent(object):
     def getEventStart(self,glueStart):
         return glueStart+self.glue.length+self.glue.adjustment
     def setOnLeaves(self,name,value): self.event.setOnLeaves(name,value)
+    def setOnLastLeaf(self,name,value): self.event.setOnLastLeaf(name,value)
 
 def setGlue(gluedEventList, schedule, glueStart = 0):
     # Uses tail recursion / backtracking with exceptions
@@ -238,9 +238,9 @@ class Lesson(object):
         else:
             # less confusing message for a beginner
             return l("Today we will learn %d words\nThis will require %d %s %d %s\nFollow the spoken instructions carefully") % (self.newWords,finish/60,singular(finish/60,"minutes"),finish%60,singular(finish%60,"seconds"))
-    def addSequence(self,gluedEventList):
+    def addSequence(self,gluedEventList,canTrack=True):
         bookIn(gluedEventList,self.schedule)
-        if not gluedListTracker==None: gluedListTracker.append(gluedEventList)
+        if not gluedListTracker==None and canTrack: gluedListTracker.append(gluedEventList)
         glueStart = 0 ; lastI = None
         for i in gluedEventList:
             i.event.setOnLeaves("sequenceID",self.eventListCounter) # for max_lateness stuff
@@ -251,7 +251,9 @@ class Lesson(object):
             if lastI: lastI.event.setOnLeaves("max_lateness",max(1,10/len(gluedEventList))+min(lastI.glue.plusMinus-lastI.glue.adjustment,i.glue.plusMinus+i.glue.adjustment))
             # (Note that this setting of max_lateness is not particularly clever - it can detect if a sequence's existing scheduling has been pushed beyond its limits, but it can't dynamically re-schedule the sequence as a whole when that happens.  Hopefully people's emergency interruptions won't be too long.)
             lastI = i
-        if lastI: lastI.event.setOnLeaves("max_lateness",max(1,10/len(gluedEventList))+lastI.glue.plusMinus-lastI.glue.adjustment)
+        if lastI:
+            lastI.event.setOnLeaves("max_lateness",max(1,10/len(gluedEventList))+lastI.glue.plusMinus-lastI.glue.adjustment)
+            if hasattr(gluedEventList[0],"timesDone"): lastI.event.setOnLastLeaf("endseq",not gluedEventList[0].timesDone)
         self.eventListCounter += 1
     def cap_max_lateness(self):
         # if an event of importance I has a max lateness of M, then all previous events with importance <I have to cap their max lateness to M+(intervening gaps) so as not to make it late.
@@ -260,7 +262,7 @@ class Lesson(object):
         latenessCap = {} ; nextStart = 0
         for t,event in self.events:
             if nextStart:
-                for k in latenessCap.keys(): latenessCap[k] += (nextStart-(t+event.length)) # the gap
+                for k in list(latenessCap.keys()): latenessCap[k] += (nextStart-(t+event.length)) # the gap
             nextStart = t
             if not hasattr(event,"importance"): continue # (wasn't added via addSequence, probably not a normal lesson)
             event.max_lateness=min(event.max_lateness,latenessCap.get(event.importance,maxLenOfLesson))
@@ -271,7 +273,8 @@ class Lesson(object):
         if (synthCache_test_mode or synthCache_test_mode==[]) and not hasattr(self,"doneSubst"):
             subst_some_synth_for_synthcache(self.events)
             self.doneSubst=1
-        global runner, finishTime, lessonLen
+        global runner, finishTime, lessonLen, wordsLeft
+        wordsLeft={False:self.oldWords,True:self.newWords}
         initLogFile()
         for (t,event) in self.events: event.will_be_played()
         if soundCollector:
@@ -285,13 +288,16 @@ class Lesson(object):
         global sequenceIDs_to_cancel ; sequenceIDs_to_cancel = {}
         global copy_of_runner_events ; copy_of_runner_events = []
         global lessonStartTime ; lessonStartTime = 0 # will be set to time.time() on 1st event
-        # make the runner as late as possible
-        if soundCollector: runner = sched.scheduler(collector_time,collector_sleep)
-        else: runner = sched.scheduler(time.time,mysleep)
-        for (t,event) in self.events: copy_of_runner_events.append((event,runner.enter(t,1,play,(event,)),t))
-        # TODO what if Brief Interrupt appears during that events loop and someone presses it (will act as a Cancel and go back to main)
-        try: runner.run()
-        except KeyboardInterrupt: handleInterrupt()
+        disable_lid(0)
+        try:
+          # make the runner as late as possible
+          if soundCollector: runner = sched.scheduler(collector_time,collector_sleep)
+          else: runner = sched.scheduler(time.time,mysleep)
+          for (t,event) in self.events: copy_of_runner_events.append((event,runner.enter(t,1,play,(event,)),t))
+          # TODO what if Brief Interrupt appears during that events loop and someone presses it (will act as a Cancel and go back to main)
+          try: runner.run()
+          except KeyboardInterrupt: handleInterrupt()
+        finally: disable_lid(1)
         runner = None
         if soundCollector: soundCollector.finished()
         if logFileHandle: logFileHandle.close()
@@ -303,17 +309,17 @@ def decide_subst_synth(cache_fname):
 def subst_some_synth_for_synthcache(events):
     # turn SOME synthcache events back into synth events (for testing new synths etc)
     reverse_transTbl = {}
-    for k,v in synthCache_transtbl.items(): reverse_transTbl[v]=k
+    for k,v in list(synthCache_transtbl.items()): reverse_transTbl[v]=k
     for i in range(len(events)):
         if hasattr(events[i][1],"file") and events[i][1].file.startswith(synthCache+os.sep):
-            cache_fname = events[i][1].file[len(synthCache+os.sep):]
+            cache_fname = B(events[i][1].file[len(synthCache+os.sep):])
             cache_fname = reverse_transTbl.get(cache_fname,cache_fname)
-            if cache_fname[0]=="_": continue # a sporadically-used synthCache entry anyway
+            if cache_fname[:1]==B("_"): continue # a sporadically-used synthCache entry anyway
             if type(synthCache_test_mode)==type([]):
                 found=0
                 for str in synthCache_test_mode:
-                    if (re and re.search(str,cache_fname)) or cache_fname.find(str)>-1:
+                    if (re and re.search(str,cache_fname)) or cache_fname.find(str)>=0:
                         found=1 ; break
                 if found: continue
             lang = languageof(cache_fname)
-            if get_synth_if_possible(lang) and decide_subst_synth(cache_fname): events[i] = (events[i][0],synth_event(lang,cache_fname[:cache_fname.rindex("_")]))
+            if get_synth_if_possible(lang) and decide_subst_synth(cache_fname): events[i] = (events[i][0],synth_event(lang,cache_fname[:cache_fname.rindex(B("_"))]))

gradint-build/src/filescan.py → src/filescan.py

-# This file is part of the source code of
-# gradint v0.9962 (c) 2002-2010 Silas S. Brown. GPL v3+.
+# This file is part of the source code of Gradint
+# (c) Silas S. Brown.
 #    This program is free software; you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by
 #    the Free Software Foundation; either version 3 of the License, or
@@ -11,15 +11,14 @@
 
 # Start of filescan.py - check for available samples and prompts and read in synthesized vocabulary
 
-limitedFiles = {} # empty this before calling scanSamples if
-                  # scanSamples ever gets called a second
-                  # time (really need to put it in an object
-                  # but...) - lists sample (RHS) filenames
-                  # that are in 'limit' dirs
-dirsWithIntros = [] # ditto
-filesWithExplanations = {} # ditto
-singleLinePoems = {} # ditto (keys are any poem files which are single line only, so as to avoid saying 'beginning' in prompts)
-variantFiles = {} # ditto (but careful if prompts is using it also)
+def init_scanSamples():
+  global limitedFiles,dirsWithIntros,filesWithExplanations,singleLinePoems,variantFiles
+  limitedFiles = {} # lists sample (RHS) filenames that are in 'limit' dirs
+  dirsWithIntros = []
+  filesWithExplanations = {}
+  singleLinePoems = {} # keys are any poem files which are single line only, so as to avoid saying 'beginning' in prompts
+  variantFiles = {} # maps dir+fname to (no dir+) fname list, main use is in fileToEvent.  Careful with clearing this if prompts is using it also (hence init_scanSamples is called only below and in loop.py before prompt scan)
+init_scanSamples() ; emptyCheck_hack = 0
 def scanSamples(directory=None):
     if not directory: directory=samplesDirectory
     # Scans the samples directory for pairs of
@@ -30,11 +29,17 @@ def scanSamples(directory=None):
     # files to use for a given "word" - currently used in
     # poetry learning)
     retVal = []
-    doLabel("Scanning samples")
+    if not emptyCheck_hack: doLabel("Scanning samples")
     if import_recordings_from: import_recordings()
     scanSamples_inner(directory,retVal,0)
     return retVal
 
+def words_exist(): # for GUI (but do NOT call from GUI thread)
+  global emptyCheck_hack ; emptyCheck_hack = 1
+  r = scanSamples() or parseSynthVocab(vocabFile)
+  emptyCheck_hack = 0
+  return r
+
 class CannotOverwriteExisting(Exception): pass
 def import_recordings(destDir=None):
     global import_recordings_from
@@ -54,7 +59,7 @@ def import_recordings(destDir=None):
                 if checkFirst:
                   for lang in [firstLanguage,secondLanguage]:
                    for ext in [dotwav,dotmp3]:
-                    if f[:f.rfind(extsep)]+"_"+lang+ext in curFiles: raise CannotOverwriteExisting()
+                    if checkIn(f[:f.rfind(extsep)]+"_"+lang+ext,curFiles): raise CannotOverwriteExisting()
                   continue
                 if not destDir:
                     if not getYN("Import the recordings that are in "+importDir+"?"): break
@@ -69,7 +74,7 @@ def import_recordings(destDir=None):
                     try:
                         import shutil
                         shutil.copy2(importDir+os.sep+f,destDir+os.sep+f)
-                    except: open(destDir+os.sep+f,"wb").write(importDir+os.sep+f).read()
+                    except: write(destDir+os.sep+f,read(importDir+os.sep+f))
                     os.remove(importDir+os.sep+f)
                 numFound += 1
     if numFound: open(destDir+os.sep+"settings"+dottxt,"w").write("firstLanguage=\""+firstLanguage+"\"\nsecondLanguage=\""+secondLanguage+"\"\n")
@@ -78,7 +83,7 @@ def import_recordings(destDir=None):
 def exec_in_a_func(x): # helper function for below (can't be nested in python 2.3)
    # Also be careful of http://bugs.python.org/issue4315 (shadowing globals in an exec) - better do this in a dictionary
    d={"firstLanguage":firstLanguage,"secondLanguage":secondLanguage}
-   exec x in d
+   exec (x,d)
    return d["secondLanguage"],d["firstLanguage"]
 def check_has_variants(directory,ls):
     if directory==promptsDirectory: return True
@@ -96,9 +101,9 @@ def getLsDic(directory):
     if not (directory.find(exclude_from_scan)==-1): return {}
     try: ls = os.listdir(directory)
     except: return {} # (can run without a 'samples' directory at all if just doing synth)
-    if "settings"+dottxt in ls:
+    if checkIn("settings"+dottxt,ls):
         # Sort out the o/p from import_recordings (and legacy record-with-HDogg.bat if anyone's still using that)
-        oddLanguage,evenLanguage = exec_in_a_func(u8strip(open(directory+os.sep+"settings"+dottxt,"rb").read().replace("\r\n","\n")).strip(wsp))
+        oddLanguage,evenLanguage = exec_in_a_func(wspstrip(u8strip(read(directory+os.sep+"settings"+dottxt).replace("\r\n","\n"))))
         if oddLanguage==evenLanguage: oddLanguage,evenLanguage="_"+oddLanguage,"-meaning_"+evenLanguage # if user sets languages the same, assume they want -meaning prompts
         else: oddLanguage,evenLanguage="_"+oddLanguage,"_"+evenLanguage
         for f in ls:
@@ -110,15 +115,15 @@ def getLsDic(directory):
             os.rename(directory+os.sep+f,directory+os.sep+f[:i]+(("%0"+str(len(str(len(ls))))+"d") % (int((int(num)-1)/2)*2+1))+cond(int(num)%2,oddLanguage,evenLanguage)+f[f.rfind(extsep):])
         os.remove(directory+os.sep+"settings"+dottxt)
         ls = os.listdir(directory)
-    ls.sort() ; lsDic = {}
+    ls.sort()
+    lsDic = {} # key is file w/out extension but INCLUDING any variant number.  Value is full filename if it's an extension we know about, "" if it's a file we can't process, or None if it's a directory (in which case key includes any 'extension' if the directory has one)
     has_variants = check_has_variants(directory,ls)
     for file in ls:
         filelower = file.lower()
-        # in lsDic if it's in the list (any extension); =filename if it's an extension we know about; =None if it's a directory (in which case the key is the full filename), ottherwise =""
-        if has_variants and file.find("_",file.find("_")+1)>-1: languageOverride=file[file.find("_")+1:file.find("_",file.find("_")+1)]
+        if filelower.endswith(dottxt) and checkIn((file+extsep)[:file.rfind(extsep)],lsDic): continue # don't let a .txt override a recording if both exist with same variant number
+        if has_variants and file.find("_",file.find("_")+1)>=0: languageOverride=file[file.find("_")+1:file.find("_",file.find("_")+1)] # for can_be_synthesized below
         else: languageOverride=None
-        if filelower.endswith(dottxt) and (file+extsep)[:file.rfind(extsep)] in lsDic: continue # don't let a .txt override a recording if both exist
-        if (filelower.endswith(dottxt) and file.find("_")>-1 and can_be_synthesized(file,directory,languageOverride)) or filelower.endswith(dotwav) or filelower.endswith(dotmp3): val = file
+        if (filelower.endswith(dottxt) and file.find("_")>=0 and can_be_synthesized(file,directory,languageOverride)) or filelower.endswith(dotwav) or filelower.endswith(dotmp3): val = file
         else:
             val = ""
             if filelower.endswith(extsep+"zip"): show_warning("Warning: Ignoring "+file+" (please unpack it first)") # so you can send someone a zip file for their recorded words folder and they'll know what's up if they don't unpack it
@@ -126,37 +131,49 @@ def getLsDic(directory):
                 lsDic[file]=None # a directory: store full name even if it has extsep in it.  Note however that we don't check isDirectory() if it's .wav etc as that would take too long.  (however some dirnames can contain dots)
                 # (+ NB need to store the directories specifically due to cases like course/ and course.pdf which may otherwise result in 2 traversals of "course" if we check isDirectory on 'extension is either none or unknown')
                 continue
-            elif (file+extsep)[:file.rfind(extsep)] in lsDic: continue # don't let a .txt~ or other unknown extension override a .txt
+            elif checkIn((file+extsep)[:file.rfind(extsep)],lsDic): continue # don't let a .txt~ or other unknown extension override a .txt
         lsDic[(file+extsep)[:file.rfind(extsep)]] = val # (this means if there's both mp3 and wav, wav will overwrite as comes later)
     if has_variants:
-        ls=list2set(ls) ; newVs = []
-        for k,v in lsDic.items():
+        ls=list2set(ls)
+        newVs = {} # variantFiles keys we added or changed
+        for k,v in list(lsDic.items()):
             # check for _lang_variant.ext and take out the _variant,
             # but keep them in variantFiles dict for fileToEvent to put back
-            if not v or (not directory==promptsDirectory and v.find("_explain_")>-1): continue # don't get confused by that
+            if not v or (not directory==promptsDirectory and v.find("_explain_")>=0): continue # skip directories, and don't get confused by explanation files
             last_ = v.rfind("_")
             if last_==-1: continue
             penult_ = v.rfind("_",0,last_)
             if penult_==-1: continue
-            del lsDic[k]
-            newK,newV = k[:k.rfind("_")], v[:v.rfind("_")]+v[v.rfind(extsep):]
-            if not newK in lsDic: lsDic[newK] = newV
-            else: # variants of different file types? better store them all under one (fileToEvent will sort out).  (Testing if the txt can be synth'd has already been done above)
-                if v.endswith(dottxt) and not lsDic[newK].endswith(dottxt): # if any variants are .txt then we'd better ensure the key is, so transliterate etc finds it. So move the key over to the .txt one.
-                    old_dirV = directory+os.sep+lsDic[newK]
+            # Now k = filename without extension but including a variant number, and v = full filename
+            del lsDic[k] # we don't want variant numbers in lsDic, we want them in variantFiles instead
+            newK,newV = k[:k.rfind("_")], v[:v.rfind("_")]+v[v.rfind(extsep):] # = k and v without the variant number (we'll add the real v to variantFiles[dir+newV] below, so it will be findable without variant number)
+            new_dirV = B(directory)+B(os.sep)+B(newV)
+            if not checkIn(newK,lsDic): # filename without variant number doesn't exist (for any extension)
+                lsDic[newK] = newV # so start it
+                assert not checkIn(new_dirV,variantFiles)
+                variantFiles[new_dirV] = [v]
+            elif v.endswith(dottxt) and not lsDic[newK].endswith(dottxt): # filename without variant number DOES exist (or we made the key when we saw a previous variant), and this new variant is .txt but the one without variant number is not.  If any variants are .txt then we'd better ensure the key maps to a .txt file (so transliterate etc finds it) and recordings are counted as variants of this .txt file, rather than .txt as variants of recordings.
+                old_dirV = B(directory+os.sep+lsDic[newK]) # the variantFiles key for the recording(s) we've already put in lsDic (but it'll be in variantFiles only if it HAD a variant number when we saw it, which won't be the case if the first variant had no number)
+                if checkIn(old_dirV,variantFiles):
                     d = variantFiles[old_dirV]
                     del variantFiles[old_dirV]
-                    lsDic[newK] = newV
-                    variantFiles[directory+os.sep+newV] = d
-                    lsDic[newK] = newV # just add to the previous key
-                else: newV = lsDic[newK]
-            dir_newV = directory+os.sep+newV
-            if not dir_newV in variantFiles:
-                variantFiles[dir_newV] = []
-                if newV in ls: variantFiles[dir_newV].append(newV) # the no-variants name is also a valid option
-            variantFiles[dir_newV].append(v)
-            newVs.append(dir_newV)
-        for v in newVs: random.shuffle(variantFiles[v])
+                    variantFiles[new_dirV] = d
+                else: variantFiles[new_dirV] = [B(lsDic[newK])] # the recording had no variant number, but now we know it does have variants, so put in the recording as first variant of the .txt key
+                variantFiles[new_dirV].append(v)
+                if checkIn(old_dirV,newVs):
+                    del newVs[old_dirV]
+                newVs[new_dirV] = 1
+                lsDic[newK] = newV
+            else: # filename without variant number does exist (or we made the key), and we need to add new variant
+                newV = lsDic[newK]
+                new_dirV = B(directory)+B(os.sep)+B(newV)
+                if not checkIn(new_dirV,variantFiles): # without variant number exists but isn't in variantFiles, so we need to add it as a variant before we add this new variant.  We know the key from lsDic.
+                    variantFiles[new_dirV] = [B(newV)]
+                variantFiles[new_dirV].append(v)
+            newVs[new_dirV]=1
+        for v in list(newVs.keys()):
+            assert checkIn(v,variantFiles), repr(sorted(list(variantFiles.keys())))+' '+repr(v)
+            random.shuffle(variantFiles[v])
     return lsDic
 
 def scanSamples_inner(directory,retVal,doLimit):
@@ -164,30 +181,33 @@ def scanSamples_inner(directory,retVal,doLimit):
     secLangSuffix = "_"+secondLanguage
     lsDic = getLsDic(directory)
     intro = intro_filename+"_"+firstLanguage
-    if intro in lsDic: dirsWithIntros.append((directory[len(samplesDirectory)+len(os.sep):],lsDic[intro]))
-    if not doLimit: doLimit = limit_filename in lsDic
-    doPoetry = poetry_filename in lsDic
+    if checkIn(intro,lsDic): dirsWithIntros.append((directory[len(samplesDirectory)+len(os.sep):],lsDic[intro]))
+    if not doLimit: doLimit = checkIn(limit_filename,lsDic)
+    doPoetry = checkIn(poetry_filename,lsDic)
     if doPoetry:
         # check which language the poetry is to be in (could be L1-to-L2, L2-to-L3, L2-only, or L3-only)
-        def poetry_language():
+        def poetry_language(firstLangSuffix,secLangSuffix,lsDic):
          ret = ""
-         for file,withExt in lsDic.items():
+         for file,withExt in list(lsDic.items()):
           if withExt:
             if file.endswith(secLangSuffix): ret=secLangSuffix # but stay in the loop
             elif (not file.endswith(firstLangSuffix)):
+                llist = [firstLanguage,secondLanguage]+otherFirstLanguages
                 for l in otherLanguages:
-                    if not l in [firstLanguage,secondLanguage] and file.endswith("_"+l): return "_"+l
+                    if not l in llist and file.endswith("_"+l): return "_"+l
          return ret
-        doPoetry = poetry_language()
+        doPoetry = poetry_language(firstLangSuffix,secLangSuffix,lsDic)
     prefix = directory[len(samplesDirectory)+cond(samplesDirectory,len(os.sep),0):] # the directory relative to samplesDirectory
     if prefix: prefix += os.sep
     lastFile = None # for doPoetry
-    items = lsDic.items() ; items.sort()
+    items = list(lsDic.items()) ; items.sort()
     for file,withExt in items:
+        swapWithPrompt = 0
         if not withExt:
             lastFile = None # avoid problems with connecting poetry lines before/after a line that's not in the synth cache or something
             if withExt==None and (cache_maintenance_mode or not directory+os.sep+file==promptsDirectory): # a directory
                 scanSamples_inner(directory+os.sep+file,retVal,doLimit)
+                if emptyCheck_hack and retVal: return
             # else no extension, or not an extension we know about - ignore (DO need this, because one way of temporarily disabling stuff is to rename it to another exension)
         elif file.find("_")==-1: continue # save confusion (!poetry, !variants etc)
         elif (doPoetry and file.endswith(doPoetry)) or (not doPoetry and (not file.endswith(firstLangSuffix) or firstLanguage==secondLanguage)): # not a prompt word
@@ -196,9 +216,10 @@ def scanSamples_inner(directory,retVal,doLimit):
                 wordSuffix=None
                 for l in otherLanguages:
                     if not l in [firstLanguage,secondLanguage] and file.endswith("_"+l):
+                        if checkIn(l,otherFirstLanguages): swapWithPrompt=1
                         wordSuffix="_"+l ; break
                 if not wordSuffix: continue # can't do anything with this file
-            if firstLanguage==secondLanguage: promptFile=None
+            if swapWithPrompt or firstLanguage==secondLanguage: promptFile=None
             else: promptFile = lsDic.get(file[:-len(wordSuffix)]+firstLangSuffix,0)
             explanationFile = lsDic.get(file[:-len(wordSuffix)]+wordSuffix+"_explain_"+firstLanguage,0)
             if not promptFile and not wordSuffix==secLangSuffix:
@@ -209,6 +230,7 @@ def scanSamples_inner(directory,retVal,doLimit):
                 promptFile = lsDic.get(file[:-len(wordSuffix)]+"-meaning"+wordSuffix,0)
             if promptFile:
                 # There is a simpler-language prompt
+                if swapWithPrompt: promptFile,withExt = withExt,promptFile
                 if doPoetry and lastFile:
                     if lastFile[0]: promptToAdd = [prefix+lastFile[0], prefix+promptFile, prefix+lastFile[1]] # both last line's and this line's prompt, then last line's contents
                     else: promptToAdd = [prefix+lastFile[1], prefix+promptFile] # last line didn't have a prompt, so put last line's contents before this line's prompt
@@ -217,15 +239,16 @@ def scanSamples_inner(directory,retVal,doLimit):
                 # poetry without first-language prompts
                 if lastFile:
                     promptToAdd = prefix+lastFile[-1]
-                    if promptToAdd in singleLinePoems: del singleLinePoems[promptToAdd]
+                    if checkIn(promptToAdd,singleLinePoems): del singleLinePoems[promptToAdd]
                 else:
                     promptToAdd = prefix+withExt # 1st line is its own prompt
                     singleLinePoems[promptToAdd]=1
             elif cache_maintenance_mode: promptToAdd = prefix+withExt
             else: continue # can't do anything with this file
             retVal.append((0,promptToAdd,prefix+withExt))
+            if emptyCheck_hack: return
             if explanationFile: filesWithExplanations[prefix+withExt]=explanationFile
-            if doLimit: limitedFiles[prefix+withExt]=prefix
+            if doLimit: limitedFiles[B(prefix+withExt)]=prefix
             lastFile = [promptFile,withExt]
 
 cache_maintenance_mode=0 # hack so cache-synth.py etc can cache promptless words for use in justSynthesize, and words in prompts themselves
@@ -233,67 +256,67 @@ def parseSynthVocab(fname,forGUI=0):
     if not fname: return []
     langs = [secondLanguage,firstLanguage] ; someLangsUnknown = 0 ; maxsplit = 1
     ret = []
-    count = 1 ; doLimit = 0 ; limitNo = 0 ; doPoetry = 0
+    count = 1 ; doLimit = 0 ; limitNo = 0 ; doPoetry = disablePoem = 0
     lastPromptAndWord = None
-    try: o=open(fname,"rb")
-    except IOError: return []
-    doLabel("Reading "+fname)
+    if not fileExists(fname): return []
+    if not emptyCheck_hack: doLabel("Reading "+fname)
     allLangs = list2set([firstLanguage,secondLanguage]+otherLanguages)
-    for l in u8strip(o.read()).replace("\r","\n").split("\n"):
+    for l in u8strip(read(fname)).replace(B("\r"),B("\n")).split(B("\n")):
         # TODO can we make this any faster on WinCE with large vocab lists? (tried SOME optimising already)
-        if not "=" in l: # might be a special instruction
+        if not B("=") in l: # might be a special instruction
             if not l: continue
-            canProcess = 0 ; l2=l.strip(wsp)
-            if not l2 or l2[0]=='#': continue
+            canProcess = 0 ; l2=bwspstrip(l)
+            if not l2 or l2[0:1]==B('#'): continue
             l2=l2.lower()
-            if l2.startswith("set language ") or l2.startswith("set languages "):
-                langs=l.split()[2:] ; someLangsUnknown = 0
+            if l2.startswith(B("set language ")) or l2.startswith(B("set languages ")):
+                langs=map(S,l.split()[2:]) ; someLangsUnknown = 0
                 maxsplit = len(langs)-1
                 for l in langs:
-                    if not l in allLangs: someLangsUnknown = 1
-            elif l2.startswith("limit on"):
+                    if not checkIn(l,allLangs): someLangsUnknown = 1
+            elif l2.startswith(B("limit on")):
                 doLimit = 1 ; limitNo += 1
-            elif l2.startswith("limit off"): doLimit = 0
-            elif l2.startswith("begin poetry"): doPoetry,lastPromptAndWord = True,None
-            elif l2.startswith("end poetry"): doPoetry = lastPromptAndWord = None
-            elif l2.startswith("poetry vocab line:"): doPoetry,lastPromptAndWord = 0,cond(lastPromptAndWord,lastPromptAndWord,0) # not None, in case we're at the very start of a poem (see "just processed"... at end)
+            elif l2.startswith(B("limit off")): doLimit = 0
+            elif l2.startswith(B("begin poetry")): doPoetry,lastPromptAndWord,disablePoem = True,None,False
+            elif l2.startswith(B("end poetry")): doPoetry = lastPromptAndWord = None
+            elif l2.startswith(B("poetry vocab line")): doPoetry,lastPromptAndWord = 0,cond(lastPromptAndWord,lastPromptAndWord,0) # not None, in case we're at the very start of a poem (see "just processed"... at end)
             else: canProcess=1
             if not canProcess: continue
-        elif "#" in l and l.strip(wsp)[0]=='#': continue # guard condition '"#" in l' improves speed
+        elif B('#') in l and bwspstrip(l)[0:1]==B('#'): continue # guard condition "'#' in l" improves speed
         if forGUI: strCount=""
         else:
             strCount = "%05d!synth:" % (count,)
             count += 1
-        langsAndWords = zip(langs,l.split("=",maxsplit)) # don't try strip on a map() - it's faster to do it as-needed below
+        langsAndWords = list(zip(langs,l.split(B("="),maxsplit))) # don't try strip on a map() - it's faster to do it as-needed below
         # (maxsplit means you can use '=' signs in the last language, e.g. if using SSML with eSpeak)
-        if someLangsUnknown: langsAndWords = filter(lambda x:x[0] in allLangs, langsAndWords)
+        if someLangsUnknown: langsAndWords = filter(lambda x,a=allLangs:checkIn(x[0],a), langsAndWords)
         # Work out what we'll use for the prompt.  It could be firstLanguage, or it could be one of the other languages if we see it twice (e.g. if 2nd language is listed twice then the second one will be the prompt for 2nd-language-to-2nd-language learning), or it could be the only language if we're simply listing words for cache maintenance
         if firstLanguage==secondLanguage: langsAlreadySeen = {}
         else: langsAlreadySeen = {firstLanguage:True}
-        def findPrompt():
+        def findPrompt(langsAndWords,langsAlreadySeen,doPoetry,strCount):
             i=0
             while i<len(langsAndWords):
                 lang,word = langsAndWords[i] ; i += 1
                 isReminder = cache_maintenance_mode and len(langsAndWords)==1 and not doPoetry
-                if (lang in langsAlreadySeen or isReminder) and (lang in getsynth_cache or can_be_synthesized("!synth:"+word+"_"+lang)): # (check cache because most of the time it'll be there and we don't need to go through all the text processing in can_be_synthesized)
+                if (lang in langsAlreadySeen or isReminder) and (lang in getsynth_cache or can_be_synthesized(B("!synth:")+B(word)+B("_")+B(lang))): # (check cache because most of the time it'll be there and we don't need to go through all the text processing in can_be_synthesized)
                     if not word: continue
-                    elif word[0] in wsp or word[-1] in wsp: word=word.strip(wsp) # avoid call if unnecessary
-                    return strCount+word+"_"+lang, cond(isReminder,0,i)
+                    elif word[0:1] in bwsp or word[-1:] in bwsp: word=bwspstrip(word) # avoid call if unnecessary
+                    return B(strCount)+word+B("_"+lang), cond(isReminder,0,i)
                 langsAlreadySeen[lang]=True
             return None,0
-        prompt,onePastPromptIndex = findPrompt()
-        if not prompt and len(langsAndWords)>1: # 1st language prompt not found; try 2nd language to 3rd language
-            langsAlreadySeen = {secondLanguage:True}
-            prompt,onePastPromptIndex = findPrompt()
+        prompt,onePastPromptIndex = findPrompt(langsAndWords,langsAlreadySeen,doPoetry,strCount)
+        if not prompt and len(langsAndWords)>1: # 1st language prompt not found; try 2nd language to 3rd language etc
+            langsAlreadySeen = list2dict(otherFirstLanguages) ; prompt,onePastPromptIndex = findPrompt(langsAndWords,langsAlreadySeen,doPoetry,strCount)
+            if not prompt:
+                langsAlreadySeen = {secondLanguage:True} ; prompt,onePastPromptIndex = findPrompt(langsAndWords,langsAlreadySeen,doPoetry,strCount)
         prompt_L1only = prompt # before we possibly change it into a list etc.  (Actually not necessarily L1 see above, but usually is)
         if doPoetry:
             if prompt and lastPromptAndWord:
-                if lastPromptAndWord[0]: prompt=[lastPromptAndWord[0],prompt,lastPromptAndWord[1]] # L1 for line 1, L1 for line2, L2 for line 1
-                else: prompt=[lastPromptAndWord[1],prompt] # line 1 doesn't have L1 but line 2 does, so have L2 for line 1 + L1 for line 2
+                if lastPromptAndWord[0]: prompt=[S(lastPromptAndWord[0]),S(prompt),S(lastPromptAndWord[1])] # L1 for line 1, L1 for line2, L2 for line 1
+                else: prompt=[S(lastPromptAndWord[1]),S(prompt)] # line 1 doesn't have L1 but line 2 does, so have L2 for line 1 + L1 for line 2
             elif not prompt:
                 if lastPromptAndWord:
                     prompt=lastPromptAndWord[-1]
-                    if lastPromptAndWord[-1] in singleLinePoems: del singleLinePoems[lastPromptAndWord[-1]]
+                    if checkIn(lastPromptAndWord[-1],singleLinePoems): del singleLinePoems[lastPromptAndWord[-1]]
                 else:
                     prompt = 1 # file itself (see below)
         if prompt:
@@ -301,44 +324,55 @@ def parseSynthVocab(fname,forGUI=0):
             while i<len(langsAndWords):
                 lang,word = langsAndWords[i] ; i+=1
                 if i==onePastPromptIndex or (lang==firstLanguage and not firstLanguage==secondLanguage) or not word: continue # if 1st language occurs more than once (target as well as prompt) then don't get confused - this vocab file is probably being used with reverse settings
-                elif word[0] in wsp or word[-1] in wsp: word=word.strip(wsp) # avoid call if unnecessary
-                if lang in getsynth_cache or can_be_synthesized("!synth:"+word+"_"+lang):
-                    f=strCount+word+"_"+lang
+                elif word[0:1] in bwsp or word[-1:] in bwsp: word=bwspstrip(word) # avoid call if unnecessary
+                if checkIn(lang,getsynth_cache) or can_be_synthesized(B("!synth:")+word+B("_"+lang)):
+                  if not (doPoetry and disablePoem):
+                    f=B(strCount)+word+B("_"+lang)
                     if prompt==1 or prompt==f: # a file with itself as the prompt (either explicitly or by omitting any other prompt)
                         prompt=f
                         singleLinePoems[f]=1
-                    ret.append((0,prompt,f))
-                    if doLimit: limitedFiles[f]="synth:"+str(limitNo)
+                    ret.append((0,S(prompt),S(f)))
+                    if emptyCheck_hack: return ret
+                    if doLimit: limitedFiles[f]=B("synth:"+str(limitNo))
                     if doPoetry: lastPromptAndWord = [prompt_L1only,f]
-                elif doPoetry: lastPromptAndWord=None # if one of the lines can't be synth'd, don't connect the lines before/after it
-        if not lastPromptAndWord==None: doPoetry = 1 # just processed a "poetry vocab line:" (lastPromptAndWord is either the real last prompt and word, or 0 if we were at the start)
+                elif doPoetry: disablePoem=1 # if one of the lines can't be synth'd, disable the rest of the poem (otherwise get wrongly connected lines, disconnected lines, or re-introduction of isolated lines that were previously part of a poem but can't be synth'd on this platform)
+        if not lastPromptAndWord==None: doPoetry = 1 # just processed a "poetry vocab line" (lastPromptAndWord is either the real last prompt and word, or 0 if we were at the start)
     return ret
 
+def sanitise_otherLanguages():
+    for l in otherFirstLanguages:
+        if not checkIn(l,otherLanguages): otherLanguages.append(l)
+    for l in otherLanguages:
+        if not checkIn(l,possible_otherLanguages): possible_otherLanguages.append(l)
+sanitise_otherLanguages()
+
 # Prompt file syntax: word_language.wav
 # or: word_language_2.wav .. (alternatives chosen at random)
 # ('word' can also be a language name)
-class PromptException(Exception):
+class MessageException(Exception):
     def __init__(self,message): self.message = message
     def __repr__(self): return self.message
+class PromptException(MessageException): pass
+auto_advancedPrompt=0 # used by gradint.cgi
 class AvailablePrompts(object):
-    reservedPrefixes = list2set(map(lambda x:x.lower(),["whatmean","meaningis","repeatAfterMe","sayAgain","longPause","begin","end",firstLanguage,secondLanguage] + otherLanguages + possible_otherLanguages))
+    reservedPrefixes = list2set(map(lambda x:x.lower(),["whatmean","meaningis","repeatAfterMe","sayAgain","longPause","begin","end",firstLanguage,secondLanguage] + possible_otherLanguages))
     def __init__(self):
         self.lsDic = getLsDic(promptsDirectory)
         self.prefixes = {}
-        for k,v in self.lsDic.items():
+        for k,v in list(self.lsDic.items()):
             if v: self.prefixes[k[:k.rfind("_")]]=1 # delete language
             else: del self.lsDic[k] # !poetry etc doesn't make sense in prompts
-        self.prefixes = self.prefixes.keys()
+        self.prefixes = list(self.prefixes.keys())
         self.user_is_advanced = None
     def getRandomPromptList(self,promptsData,language):
         random.shuffle(self.prefixes)
         for p in self.prefixes:
-            if p.lower() in self.reservedPrefixes: continue
+            if checkIn(p.lower(),self.reservedPrefixes): continue
             try:
                 theList = self.getPromptList(p,promptsData,language)
                 return theList
             except PromptException: pass
-        raise PromptException("Can't find a non-reserved prompt suitable for language '%s'" % (language))
+        raise PromptException("Can't find a non-reserved prompt suitable for language '%s'. Try creating tryToSay_%s%s etc in %s" % (language,language,dotwav,promptsDirectory))
     def getPromptList(self,prefix,promptsData,language):
         # used for introducing foreign-language prompts to
         # beginners.  language is the suffix of the language we're *learning*.
@@ -348,11 +382,11 @@ class AvailablePrompts(object):
                 if p > advancedPromptThreshold2:
                     self.user_is_advanced = 1 ; break # got a reasonably advanced user
         beginnerPrompt = prefix+"_"+firstLanguage
-        if not beginnerPrompt in self.lsDic:
+        if not checkIn(beginnerPrompt,self.lsDic):
             if self.user_is_advanced and not language==secondLanguage and prefix+"_"+secondLanguage in self.lsDic: beginnerPrompt=prefix+"_"+secondLanguage # No first language prompt, but in advanced mode may be able to find a second-language prompt for a 3rd language
             else: beginnerPrompt = None
         advancedPrompt = prefix+"_"+language
-        if not advancedPrompt in self.lsDic:
+        if not checkIn(advancedPrompt,self.lsDic):
             # Must use beginnerPrompt
             if beginnerPrompt: r=[self.lsDic[beginnerPrompt]]
             else:
@@ -360,7 +394,7 @@ class AvailablePrompts(object):
                 else: raise PromptException("Can't find "+prefix+"_"+language+", "+prefix+"_"+firstLanguage+" or "+prefix+"_"+secondLanguage)
         elif not beginnerPrompt:
             # Must use advancedPrompt
-            if (not self.user_is_advanced) and cond(language==secondLanguage,advancedPromptThreshold,advancedPromptThreshold2): raise PromptException("Prompt '%s' is too advanced; need '%s_%s' (unless you set %s=0 in advanced%stxt)" % (advancedPrompt,prefix,firstLanguage,cond(language==secondLanguage,"advancedPromptThreshold","advancedPromptThreshold2"),extsep))
+            if (not self.user_is_advanced) and not auto_advancedPrompt and cond(language==secondLanguage,advancedPromptThreshold,advancedPromptThreshold2): raise PromptException("Prompt '%s' is too advanced; need '%s_%s' (unless you set %s=0 in advanced%stxt)" % (advancedPrompt,prefix,firstLanguage,cond(language==secondLanguage,"advancedPromptThreshold","advancedPromptThreshold2"),extsep))
             r=[self.lsDic[advancedPrompt]]
         elif promptsData.get(advancedPrompt,0) >= cond(language==secondLanguage,advancedPromptThreshold,advancedPromptThreshold2): r=[self.lsDic[advancedPrompt]]
         elif promptsData.get(advancedPrompt,0) >= cond(language==secondLanguage,transitionPromptThreshold,transitionPromptThreshold2): r=[self.lsDic[advancedPrompt], self.lsDic[beginnerPrompt]]
@@ -373,27 +407,27 @@ class AvailablePrompts(object):
         # Increment advancedPrompt, taking care not to go
         # past the threshold if it's not available yet
         adv = promptsData.get(advancedPrompt,0)
-        if advancedPrompt in self.lsDic or adv <= cond(language==secondLanguage,transitionPromptThreshold,transitionPromptThreshold2):
+        if checkIn(advancedPrompt,self.lsDic) or adv <= cond(language==secondLanguage,transitionPromptThreshold,transitionPromptThreshold2):
             adv += 1
         promptsData[advancedPrompt] = adv
         # and finally,
         if not language==secondLanguage and not prefix==language and not prefix=="meaningis": r=self.getPromptList(language,promptsData,language)+r # yes, before - works better than after
         return r
-# availablePrompts = AvailablePrompts() # do NOT construct here - if a warning is printed (e.g. can't find a synth) then it might go to the wrong place if GUI has not yet started.  Constructor moved to lesson_loop().
+# Do NOT construct availablePrompts here - if a warning is printed (e.g. can't find a synth) then it might go to the wrong place if GUI has not yet started.  Constructing moved to lesson_loop().
 
 def introductions(zhFile,progressData):
     toIntroduce = []
     for d,fname in dirsWithIntros[:]:
         found = 0
         for p in progressData:
-            if p[-1].startswith(d) and p[0]:
+            if B(p[-1]).startswith(B(d)) and p[0]:
                 # this dir has already been introduced
                 found=1 ; dirsWithIntros.remove((d,fname)) ; break
         if found: continue
-        if zhFile.startswith(d): toIntroduce.append((d,fname))
+        if B(zhFile).startswith(B(d)): toIntroduce.append((d,fname))
     toIntroduce.sort() # should put shorter ones 1st
-    return map(lambda (x,fname): fileToEvent(cond(x,x+os.sep,"")+fname), toIntroduce)
+    return map(lambda x: fileToEvent(cond(x[0],x[0]+os.sep,"")+x[1]), toIntroduce)
 
 def explanations(zhFile):
-    if zhFile in filesWithExplanations: return fileToEvent(zhFile.replace(dotmp3,dotwav).replace(dottxt,dotwav).replace(dotwav,"_explain_"+firstLanguage+filesWithExplanations[zhFile][-len(dotwav):]))
+    if checkIn(zhFile,filesWithExplanations): return fileToEvent(zhFile.replace(dotmp3,dotwav).replace(dottxt,dotwav).replace(dotwav,"_explain_"+firstLanguage+filesWithExplanations[zhFile][-len(dotwav):]))
 

gradint-build/src/frontend.py → src/frontend.py

-# This file is part of the source code of
-# gradint v0.9962 (c) 2002-2010 Silas S. Brown. GPL v3+.
+# This file is part of the source code of Gradint
+# (c) Silas S. Brown.
 #    This program is free software; you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by
 #    the Free Software Foundation; either version 3 of the License, or
@@ -12,7 +12,7 @@
 # Start of frontend.py - Tk and other front-ends
 
 def interrupt_instructions():
-    if soundCollector or app or appuifw: return ""
+    if soundCollector or app or appuifw or android: return ""
     elif msvcrt: return "\nPress Space if you have to interrupt the lesson."
     elif riscos_sound: return "\nLesson interruption not yet implemented on RISC OS.  If you stop the program before the end of the lesson, your progress will be lost.  Sorry about that."
     elif winCEsound: return "\nLesson interruption not implemented on\nWinCE without GUI.  Can't stop, sorry!"
@@ -23,18 +23,29 @@ appTitle += time.strftime(" %A") # in case leave 2+ instances on the desktop
 def waitOnMessage(msg):
     global warnings_printed
     if type(msg)==type(u""): msg2=msg.encode("utf-8")
-    else: msg2=msg
+    else:
+        try: msg2,msg=msg,msg.decode("utf-8")
+        except AttributeError: msg2=msg # Python 2.1 has no .decode
     if appuifw:
         t=appuifw.Text() ; t.add(u"".join(warnings_printed)+msg) ; appuifw.app.body = t # in case won't fit in the query()  (and don't use note() because it doesn't wait)
-        appuifw.query(u""+msg,'query')
+        appuifw.query(msg,'query')
+    elif android:
+        # android.notify("Gradint","".join(warnings_printed)+msg) # doesn't work?
+        android.dialogCreateAlert("Gradint","".join(warnings_printed)+msg)
+        android.dialogSetPositiveButtonText("OK")
+        android.dialogShow() ; android.dialogGetResponse()
     elif app:
-        if not (winsound or winCEsound or mingw32 or cygwin): show_info(msg2+"\n\nWaiting for you to press OK on the message box... ",True) # in case terminal is in front
+        if not (winsound or winCEsound or mingw32 or cygwin): show_info(msg2+B("\n\nWaiting for you to press OK on the message box... "),True) # in case terminal is in front
         app.todo.alert = "".join(warnings_printed)+msg
-        while hasattr(app.todo,"alert"): time.sleep(0.5)
+        while True:
+            try:
+              if not hasattr(app.todo,"alert"): break
+            except: break # app destroyed
+            time.sleep(0.5)
         if not (winsound or winCEsound or mingw32 or cygwin): show_info("OK\n",True)
     else:
-        if clearScreen(): msg2 = "This is "+program_name.replace("(c)","\n(c)")+"\n\n"+msg2 # clear screen is less confusing for beginners, but NB it may not happen if warnings etc
-        show_info(msg2+"\n\n"+cond(winCEsound,"Press OK to continue\n","Press Enter to continue\n"))
+        if clearScreen(): msg2 = B("This is "+program_name.replace("(c)","\n(c)")+"\n\n")+msg2 # clear screen is less confusing for beginners, but NB it may not happen if warnings etc
+        show_info(msg2+B("\n\n"+cond(winCEsound,"Press OK to continue\n","Press Enter to continue\n")))
         sys.stderr.flush() # hack because some systems don't do it (e.g. some mingw32 builds), and we don't want the user to fail to see why the program is waiting (especially when there's an error)
         try:
             raw_input(cond(winCEsound,"See message under this window.","")) # (WinCE uses boxes for raw_input so may need to repeat the message - but can't because the prompt is size-limited, so need to say look under the window)
@@ -45,10 +56,18 @@ def waitOnMessage(msg):
 def getYN(msg,defaultIfEof="n"):
     if appuifw:
         appuifw.app.body = None
-        return appuifw.query(u""+msg,'query')
+        return appuifw.query(ensure_unicode(msg),'query')
+    elif android:
+        android.dialogCreateAlert("Gradint",msg)
+        android.dialogSetPositiveButtonText("Yes") # TODO do we have to localise this ourselves or can we have a platform default?
+        android.dialogSetNegativeButtonText("No")
+        android.dialogShow()
+        try: return android.dialogGetResponse().result['which'] == 'positive'
+        except KeyError: return 0 # or raise SystemExit, no 'which'
     elif app:
         app.todo.question = localise(msg)
-        while not hasattr(app,"answer_given"): time.sleep(0.5)
+        while app and not hasattr(app,"answer_given"): time.sleep(0.5)
+        if not app: raise SystemExit
         ans = app.answer_given
         del app.answer_given
         return ans
@@ -58,54 +77,70 @@ def getYN(msg,defaultIfEof="n"):
         while not ans=='y' and not ans=='n':
             try: ans = raw_input("%s\nPress y for yes, or n for no.  Then press Enter.  --> " % (msg,))
             except EOFError:
-                ans=defaultIfEof ; print ans
+                ans=defaultIfEof ; print (ans)
         clearScreen() # less confusing for beginners
         if ans=='y': return 1
         return 0
 
 def primitive_synthloop():
+    global justSynthesize,warnings_printed
     lang = None
+    interactive = appuifw or winCEsound or android or not hasattr(sys.stdin,"isatty") or sys.stdin.isatty()
+    if interactive: interactive=cond(winCEsound and warnings_printed,"(see warnings under this window) Say:","Say: ") # (WinCE uses an input box so need to repeat the warnings if any - but can't because prompt is size-limited, so need to say move the window.)
+    else: interactive="" # no prompt on the raw_input (we might be doing outputFile="-" as well)
     while True:
-        global justSynthesize,warnings_printed
+        old_js = justSynthesize
         if appuifw:
             if not justSynthesize: justSynthesize=""
-            justSynthesize=appuifw.query(u"Say:","text",u""+justSynthesize)
+            justSynthesize=appuifw.query(u"Say:","text",ensure_unicode(justSynthesize))
             if justSynthesize: justSynthesize=justSynthesize.encode("utf-8")
             else: break
         else:
-            try: justSynthesize=raw_input(cond(winCEsound and warnings_printed,"(see warnings under this window) Say:","Say: ")) # (WinCE uses an input box so need to repeat the warnings if any - but can't because prompt is size-limited, so need to say move the window.)
-            except EOFError: break
-            if (winCEsound or riscos_sound) and not justSynthesize: break # because no way to send EOF (and we won't be taking i/p from a file)
+            if android:
+              justSynthesize = android.dialogGetInput("Gradint",interactive).result
+              if type(justSynthesize)==type(u""): justSynthesize=justSynthesize.encode("utf-8")
+            else:
+              try: justSynthesize=raw_input(interactive)
+              except EOFError: break
+            if (winCEsound or riscos_sound or android) and not justSynthesize: break # because no way to send EOF (and we won't be taking i/p from a file)
+            if interactive and not readline:
+              interactive="('a' for again) Say: "
+              if B(justSynthesize)==B("a"): justSynthesize=old_js
         oldLang = lang
-        if justSynthesize: lang = just_synthesize(appuifw or not hasattr(sys.stdin,"isatty") or sys.stdin.isatty(),lang)
+        if justSynthesize: lang = S(just_synthesize(interactive,lang))
         # and see if it transliterates:
-        if justSynthesize and lang and not "#" in justSynthesize:
-            if justSynthesize.startswith(lang+" "):
+        if justSynthesize and lang and not B('#') in B(justSynthesize):
+            if B(justSynthesize).startswith(B(lang)+B(" ")):
                 t = transliterates_differently(justSynthesize[len(lang+" "):],lang)
                 if t: t=lang+" "+t
             else: t = transliterates_differently(justSynthesize,lang)
             if t:
                 if appuifw: justSynthesize = t
-                else: show_info("Spoken as "+t+"\n")
+                else: show_info(B("Spoken as ")+t+B("\n"))
         if warnings_printed: # at end not beginning, because don't want to overwrite the info message if appuifw
             if appuifw:
                 t=appuifw.Text()
                 t.add(u"".join(warnings_printed))
                 appuifw.app.body = t
+            elif android: waitOnMessage("") # (makeToast doesn't stay around for very long)
             # else they'll have already been printed
             warnings_printed = []
         if not lang: lang=oldLang
 
+if android:
+  if not isDirectory("/mnt/sdcard/svox") and not isDirectory("/system/tts/lang_pico"): waitOnMessage("English voice might not be installed. Check under Home > Menu > Settings > Voice output > text to speech > Pico > English")
+
 def startBrowser(url): # true if success
   if winCEsound: return None # user might be paying per byte! + difficult to switch back if no Alt-Tab program
-  try: import webbrowser
-  except: webbrowser=0
-  if webbrowser:
+  try:
+      import webbrowser
       g=webbrowser.get()
-      if g and (winCEsound or macsound or (hasattr(g,"background") and g.background) or (hasattr(webbrowser,"BackgroundBrowser") and g.__class__==webbrowser.BackgroundBrowser) or (hasattr(webbrowser,"Konqueror") and g.__class__==webbrowser.Konqueror)):
-          return g.open_new(url)
-      # else don't risk it - it might be text-mode and unsuitable for multitask-with-gradint
+  except: g=0
+  if g and (winCEsound or macsound or (hasattr(g,"background") and g.background) or (hasattr(webbrowser,"BackgroundBrowser") and g.__class__==webbrowser.BackgroundBrowser) or (hasattr(webbrowser,"Konqueror") and g.__class__==webbrowser.Konqueror)):
+      return g.open_new(S(url))
+  # else don't risk it - it might be text-mode and unsuitable for multitask-with-gradint
   if winsound: return not os.system('start "%ProgramFiles%\\Internet Explorer\\iexplore.exe" '+url) # use os.system not system here (don't know why but system() doesn't always work for IE)
+  # (NB DON'T replace % with %%, it doesn't work. just hope nobody set an environment variable to any hex code we're using in mp3web)
 
 def clearScreen():
     global warnings_printed
@@ -115,7 +150,7 @@ def clearScreen():
         warnings_printed = []
         return
     if winsound or mingw32: os.system("cls")
-    else: os.system("clear 1>&2") # (1>&2 in case using stdout for something else)
+    else: os.system("clear >&2") # (>&2 in case using stdout for something else)
     return True
 
 cancelledFiles = []
@@ -135,17 +170,17 @@ def handleInterrupt(): # called only if there was an interrupt while the runner
         # cancelledEvent = runner.queue[0][-1][0] worked in python 2.3, but sched implementation seems to have changed in python 2.5 so we're using copy_of_runner_events instead
         if hasattr(cancelledEvent,"wordToCancel") and cancelledEvent.wordToCancel: cancelledFiles.append(cancelledEvent.wordToCancel)
     if not app and needCountItems and cancelledFiles: show_info("(%d cancelled items)...\n" % len(cancelledFiles))
-    global repeatMode ; repeatMode = 0 # so Ctrl-C on justSynth-with-R works
+    global repeatMode ; repeatMode = "interrupted"
 
 tkNumWordsToShow = 10 # the default number of list-box items
 
 def addStatus(widget,status,mouseOnly=0):
     # Be VERY CAREFUL with status line changes.  Don't do it on things that are focused by default (except with mouseOnly=1).  Don't do it when the default status line might be the widest thing (i.e. when list box is not displayed) or window size could jump about too much.  And in any event don't use lines longer than about 53 characters (the approx default width of the listbox when using monospace fonts).
     # (NB addStatus now takes effect only when the list box is displayed anyway, so OK for buttons that might also be displayed without it)
-    widget.bind('<Enter>',lambda *args:app.set_statusline(status))
+    widget.bind('<Enter>',lambda e=None,status=status:app.set_statusline(status))
     widget.bind('<Leave>',app.restore_statusline)
     if not mouseOnly:
-        widget.bind('<FocusIn>',lambda *args:app.set_statusline(status))
+        widget.bind('<FocusIn>',lambda e=None,app=app,status=status:app.set_statusline(status))
         widget.bind('<FocusOut>',app.restore_statusline)
 def makeButton(parent,text,command):
     button = Tkinter.Button(parent)
@@ -159,6 +194,7 @@ def addButton(parent,text,command,packing=None,status=None):
     button = makeButton(parent,text,command)
     if status: addStatus(button,status)
     if packing=="nopack": pass
+    elif type(packing)==type(""): button.pack(side=packing)
     elif packing: button.pack(packing)
     else: button.pack()
     return button
@@ -169,12 +205,17 @@ def addLabel(row,label):
 def CXVMenu(e): # callback for right-click
     e.widget.focus()
     m=Tkinter.Menu(None, tearoff=0, takefocus=0)
-    ctrl=cond(macsound,"<Command-","<Control-")
-    m.add_command(label="Cut",command=(lambda e=e: e.widget.event_generate(ctrl+'x>')))
-    m.add_command(label="Copy",command=(lambda e=e: e.widget.event_generate(ctrl+'-c>')))
-    m.add_command(label="Paste",command=(lambda e=e: e.widget.event_generate(ctrl+'-v>')))
-    m.add_command(label="Delete",command=(lambda e=e: e.widget.event_generate('<Delete>')))
-    m.add_command(label="Select All",command=(lambda e=e: selectAll(e)))
+    if macsound:
+        cut,copy,paste = "<<Cut>>","<<Copy>>","<<Paste>>"
+    else:
+        ctrl="<Control-"
+        cut,copy,paste = ctrl+'x>',ctrl+'c>',ctrl+'v>'
+    def evgen(e,cmd): e.widget.event_generate(cmd)
+    funclist = [("Paste",paste),("Delete",'<Delete>')]
+    if not macsound:
+        funclist = [("Cut",cut),("Copy",copy)]+funclist # doesn't work reliably on Mac Tk
+    for l,cmd in funclist: m.add_command(label=l,command=(lambda e=e,c=cmd,evgen=evgen: e.widget.after(10,evgen,e,c)))
+    m.add_command(label="Select All",command=(lambda e=e: e.widget.after(10,selectAll,e)))
     m.tk_popup(e.x_root-3, e.y_root+3,entry="0")
 def selectAll(e):
     e.widget.event_generate('<Home>')
@@ -187,26 +228,28 @@ def selectAllButNumber(e): # hack for recording.py - select all but any number a
 def addTextBox(row,wide=0):
     text = Tkinter.StringVar(row)
     entry = Tkinter.Entry(row,textvariable=text)
-    if winsound or mingw32 or cygwin or macsound: entry.bind('<Button-3>',CXVMenu)
-    if macsound: entry.bind('<Control-Button-1>',CXVMenu)
+    entry.bind('<ButtonRelease-3>',CXVMenu)
+    if macsound:
+        entry.bind('<Control-ButtonRelease-1>',CXVMenu)
+        entry.bind('<ButtonRelease-2>',CXVMenu)
     if winCEsound:
       if WMstandard: # non-numeric inputs no good on WMstandard Tkinter
         def doRawInput(text,entry):
             app.input_to_set = text
             app.menu_response="input"
-        entry.bind('<Return>',lambda e:doRawInput(text,entry))
+        entry.bind('<Return>',lambda e,doRawInput=doRawInput,text=text,entry=entry:doRawInput(text,entry))
         if wide: # put help in 1st wide textbox
           global had_doRawInput
           try: had_doRawInput
           except:
             had_doRawInput=1
-            text.set("(Push OK to type A-Z)")
+            text.set("(Push OK to type A-Z)") # (if changing this message, change it below too)
             class E: pass
             e=E() ; e.widget = entry
-            entry.after(10,lambda *args:selectAll(e))
+            entry.after(10,lambda _=None,e=e:selectAll(e))
       else: # PocketPC: try to detect long clicks. This is awkward. time.time is probably 1sec resolution so will get false +ves if go by that only.
         def timeStamp(entry): entry.buttonPressTime=time.time()
-        entry.bind('<ButtonPress-1>',lambda e:timeStamp(entry))
+        entry.bind('<ButtonPress-1>',lambda e,timeStamp=timeStamp,entry=entry:timeStamp(entry))
         global lastDblclkAdvisory,lastDblclk
         lastDblclkAdvisory=lastDblclk=0
         def pasteInstructions(t):
@@ -218,8 +261,8 @@ def addTextBox(row,wide=0):
         def doPaste(text,entry):
             text.set(entry.selection_get(selection="CLIPBOARD"))
             global lastDblclk ; lastDblclk=time.time()
-        entry.bind('<ButtonRelease-1>',lambda e:pasteInstructions(time.time()-getattr(entry,"buttonPressTime",time.time())))
-        entry.bind('<Double-Button-1>',lambda e:doPaste(text,entry))
+        entry.bind('<ButtonRelease-1>',lambda e,entry=entry,pasteInstructions=pasteInstructions:pasteInstructions(time.time()-getattr(entry,"buttonPressTime",time.time())))
+        entry.bind('<Double-Button-1>',lambda e,doPaste=doPaste,text=text,entry=entry:doPaste(text,entry))
     # Tkinter bug workaround (some versions): event_generate from within a key event handler can be unreliable, so the Ctrl-A handler delays selectAll by 10ms:
     entry.bind(cond(macsound,'<Command-a>','<Control-a>'),(lambda e:e.widget.after(10,lambda e=e:selectAll(e))))
     bindUpDown(entry,False)
@@ -256,24 +299,33 @@ def make_output_row(parent):
     # if there aren't any options then return None
     # we also put script-variant selection here, if any
     row = None
-    if "@variants-"+firstLanguage in GUI_translations: # the firstLanguage has script variants
+    def getRow(row,parent):
+      if not row:
         row = Tkinter.Frame(parent)
         row.pack(fill=Tkinter.X,expand=1)
+      return row
+    GUIlang = GUI_languages.get(firstLanguage,firstLanguage)
+    if checkIn("@variants-"+GUIlang,GUI_translations): # the firstLanguage has script variants
+        row=getRow(row,parent)
         if not hasattr(app,"scriptVariant"): app.scriptVariant = Tkinter.StringVar(app)
         count = 0
-        for variant in GUI_translations["@variants-"+firstLanguage]:
+        for variant in GUI_translations["@variants-"+GUIlang]:
             Tkinter.Radiobutton(row, text=u" "+variant+u" ", variable=app.scriptVariant, value=str(count), indicatoron=forceRadio).pack({"side":"left"})
             count += 1
-        app.scriptVariant.set(str(scriptVariants.get(firstLanguage,0)))
+        app.scriptVariant.set(str(scriptVariants.get(GUIlang,0)))
+    if synth_partials_voices and guiVoiceOptions:
+        row=getRow(row,parent)
+        if not hasattr(app,"voiceOption"): app.voiceOption = Tkinter.StringVar(app)
+        Tkinter.Radiobutton(row, text=u" Normal ", variable=app.voiceOption, value="", indicatoron=forceRadio).pack({"side":"left"})
+        for o in guiVoiceOptions: Tkinter.Radiobutton(row, text=u" "+o[1].upper()+o[2:]+u" ", variable=app.voiceOption, value=o, indicatoron=forceRadio).pack({"side":"left"})
+        app.voiceOption.set(voiceOption)
     if not gotSox: return row # can't do any file output without sox
     if not hasattr(app,"outputTo"):
         app.outputTo = Tkinter.StringVar(app) # NB app not parent (as parent is no longer app)
         app.outputTo.set("0") # not "" or get tri-state boxes on OS X 10.6
-    if not row:
-        row = Tkinter.Frame(parent)
-        row.pack(fill=Tkinter.X,expand=1)
+    row=getRow(row,parent)
     rightrow = addRightRow(row) # to show beginners this row probably isn't the most important thing despite being in a convenient place, we'll right-align
-    def addFiletypeButton(fileType):
+    def addFiletypeButton(fileType,rightrow):
         ftu = fileType.upper()
         t = Tkinter.Radiobutton(rightrow, text=cond(forceRadio,""," ")+ftu+" ", variable=app.outputTo, value=fileType, indicatoron=forceRadio)
         bindUpDown(t,True)
@@ -287,34 +339,34 @@ def make_output_row(parent):
     addStatus(t,"Select this to send all sounds to\nthe speaker, not to files on disk")
     bindUpDown(t,True)
     t.pack({"side":"left"})
-    if got_program("lame"): addFiletypeButton("mp3")
-    if got_windows_encoder: addFiletypeButton("wma")
-    if got_program("faac") or got_program("afconvert"): addFiletypeButton("aac")
-    if got_program("oggenc"): addFiletypeButton("ogg")
-    if got_program("toolame"): addFiletypeButton("mp2")
-    if got_program("speexenc"): addFiletypeButton("spx")
-    addFiletypeButton("wav")
+    if got_program("lame"): addFiletypeButton("mp3",rightrow)
+    if got_windows_encoder: addFiletypeButton("wma",rightrow)
+    if got_program("neroAacEnc") or got_program("faac") or got_program("afconvert"): addFiletypeButton("aac",rightrow)
+    if got_program("oggenc") or got_program("oggenc2"): addFiletypeButton("ogg",rightrow)
+    if got_program("toolame"): addFiletypeButton("mp2",rightrow)
+    if got_program("speexenc"): addFiletypeButton("spx",rightrow)
+    addFiletypeButton("wav",rightrow)
     # "Get MP3 encoder" and "Get WMA encoder" changed to "MP3..." and "WMA..." to save width (+ no localisation necessary)
     if unix and not got_program("lame") and got_program("make") and got_program("gcc") and (got_program("curl") or got_program("wget")): addButton(rightrow,"MP3...",app.getEncoder,status="Press this to compile an MP3 encoder\nso Gradint can output to MP3 files") # (checking gcc as well as make because some distros strangely have make but no compiler; TODO what if has a non-gcc compiler)
-    elif (winsound or mingw32) and not got_windows_encoder and not got_program("lame"): addButton(rightrow,"WMA...",app.getEncoder,status="Press this to download a WMA encoder\nso Gradint can output to WMA files")
+    # (no longer available) elif (winsound or mingw32) and not got_windows_encoder and not got_program("lame"): addButton(rightrow,"WMA...",app.getEncoder,status="Press this to download a WMA encoder\nso Gradint can output to WMA files")
     return row
 
 def updateSettingsFile(fname,newVals):
     # leaves comments etc intact, but TODO does not cope with changing variables that have been split over multiple lines
     replacement_lines = []
-    try: oldLines=u8strip(open(fname,"rb").read()).replace("\r\n","\n").split("\n")
+    try: oldLines=u8strip(read(fname)).replace(B("\r\n"),B("\n")).split(B("\n"))
     except IOError: oldLines=[]
     for l in oldLines:
         found=0
-        for k in newVals.keys():
-            if l.startswith(k):
-                replacement_lines.append(k+"="+repr(newVals[k]))
+        for k in list(newVals.keys()):
+            if l.startswith(B(k)):
+                replacement_lines.append(B(k+"="+repr(newVals[k])))
                 del newVals[k]
                 found=1
         if not found: replacement_lines.append(l)
-    for k,v in newVals.items(): replacement_lines.append(k+"="+repr(v))
-    if replacement_lines and replacement_lines[-1]: replacement_lines.append("") # ensure blank line at end so there's a \n but we don't add 1 more with each save
-    open(fname,"w").write("\n".join(replacement_lines))
+    for k,v in list(newVals.items()): replacement_lines.append(B(k+"="+repr(v)))
+    if replacement_lines and replacement_lines[-1]: replacement_lines.append(B("")) # ensure blank line at end so there's a \n but we don't add 1 more with each save
+    writeB(open(fname,"w"),B("\n").join(replacement_lines))
 
 def asUnicode(x): # for handling the return value of Tkinter entry.get()
     try: return u""+x # original behaviour
@@ -323,16 +375,27 @@ def asUnicode(x): # for handling the return value of Tkinter entry.get()
         except: return x.decode("iso-8859-1") # TODO can we get what it actually IS? (on German WinXP, sys.getdefaultencoding==ascii and locale==C but Tkinter still returns Latin1)
 
 def setupScrollbar(parent,rowNo):
+    onLeft = winCEsound or olpc
     s = Tkinter.Scrollbar(parent,takefocus=0)
-    s.grid(row=rowNo,column=cond(winCEsound or olpc,0,1),sticky="ns"+cond(winCEsound or olpc,"w","e"))
+    s.grid(row=rowNo,column=cond(onLeft,0,1),sticky="ns"+cond(onLeft,"w","e"))
+    try: parent.rowconfigure(rowNo,weight=1)
+    except: pass
     c=Tkinter.Canvas(parent,bd=0,width=200,height=100,yscrollcommand=s.set)
-    c.grid(row=rowNo,column=cond(winCEsound or olpc,1,0),sticky="nsw")
+    c.grid(row=rowNo,column=cond(onLeft,1,0),sticky="nsw")
     s.config(command=c.yview)
     scrolledFrame=Tkinter.Frame(c) ; c.create_window(0,0,window=scrolledFrame,anchor="nw")
     # Mousewheel binding.  TODO the following bind_all assumes only one scrolledFrame on screen at once (redirect all mousewheel events to the frame; necessary as otherwise they'll go to buttons etc)
-    scrolledFrame.bind_all('<Button-4>',lambda *args:c.yview("scroll","-1","units"))
-    scrolledFrame.bind_all('<Button-5>',lambda *args:c.yview("scroll","1","units"))
-    # DON'T bind <MouseWheel> on Windows - our version of Tk will segfault when it occurs. See http://mail.python.org/pipermail/python-bugs-list/2005-May/028768.html but we can't patch our library.zip's Tkinter anymore (TODO can we use newer Tk DLLs and ensure setup.bat updates them?)
+    app.ScrollUpHandler = lambda e=None,c=c:c.yview("scroll","-1","units")
+    app.ScrollDownHandler = lambda e=None,c=c:c.yview("scroll","1","units")
+    if macsound:
+        def ScrollHandler(event):
+            if event.delta>0: app.ScrollUpHandler()
+            else: app.ScrollDownHandler()
+        scrolledFrame.bind_all('<MouseWheel>',ScrollHandler)
+        # DON'T bind <MouseWheel> on Windows - our version of Tk will segfault when it occurs. See http://mail.python.org/pipermail/python-bugs-list/2005-May/028768.html but we can't patch our library.zip's Tkinter anymore (TODO can we use newer Tk DLLs and ensure setup.bat updates them?)
+    else: # for X11:
+        scrolledFrame.bind_all('<Button-4>',app.ScrollUpHandler)
+        scrolledFrame.bind_all('<Button-5>',app.ScrollDownHandler)
     return scrolledFrame, c
 
 # GUI presets buttons:
@@ -340,8 +403,8 @@ shortDescriptionName = "short-description"+dottxt
 longDescriptionName = "long-description"+dottxt
 class ExtraButton(object):
     def __init__(self,directory):
-        self.shortDescription = u8strip(open(directory+os.sep+shortDescriptionName).read()).strip(wsp)
-        if fileExists(directory+os.sep+longDescriptionName): self.longDescription = u8strip(open(directory+os.sep+longDescriptionName).read()).strip(wsp)
+        self.shortDescription = wspstrip(u8strip(read(directory+os.sep+shortDescriptionName)))
+        if fileExists(directory+os.sep+longDescriptionName): self.longDescription = wspstrip(u8strip(read(directory+os.sep+longDescriptionName)))
         else: self.longDescription = self.shortDescription
         self.directory = directory
     def add(self):
@@ -358,7 +421,7 @@ class ExtraButton(object):
         try: ls = os.listdir(samplesDirectory)
         except: os.mkdir(samplesDirectory)
         name1=newName
-        while newName in ls: newName+="1"
+        while checkIn(newName,ls): newName+="1"
         name2=newName
         newName = samplesDirectory+os.sep+newName
         os.rename(self.directory,newName)
@@ -367,18 +430,16 @@ class ExtraButton(object):
             which_collection = localise(" has been added to your collection.")
             o=open(vocabFile,"a")
             o.write("# --- BEGIN "+self.shortDescription+" ---\n")
-            o.write(u8strip(open(newName+os.sep+"add-to-vocab"+dottxt,"rb").read()).strip(wsp)+"\n")
+            o.write(wspstrip(u8strip(read(newName+os.sep+"add-to-vocab"+dottxt)))+"\n")
             o.write("# ----- END "+self.shortDescription+" ---\n")
             if hasattr(app,"vocabList"): del app.vocabList # so re-reads
             os.remove(newName+os.sep+"add-to-vocab"+dottxt)
         if fileExists(newName+os.sep+"add-to-languages"+dottxt):
             changed = 0
-            for lang in u8strip(open(newName+os.sep+"add-to-languages"+dottxt,"rb").read()).strip(wsp).split():
+            for lang in wspstrip(u8strip(read(newName+os.sep+"add-to-languages"+dottxt))).split():
                 if not lang in [firstLanguage,secondLanguage]+otherLanguages:
                     otherLanguages.append(lang) ; changed = 1
-                if not lang in possible_otherLanguages:
-                    possible_otherLanguages.append(lang) ; changed = 1
-            if changed: updateSettingsFile("advanced"+dottxt,{"otherLanguages":otherLanguages,"possible_otherLanguages":possible_otherLanguages})
+            if changed: sanitise_otherLanguages(), updateSettingsFile("advanced"+dottxt,{"otherLanguages":otherLanguages,"possible_otherLanguages":possible_otherLanguages})
             os.remove(newName+os.sep+"add-to-languages"+dottxt)
         promptsAdd = newName+os.sep+"add-to-prompts"
         if isDirectory(promptsAdd):
@@ -390,6 +451,7 @@ class ExtraButton(object):
         self.button.pack_forget()
         app.extra_button_callables.remove(self)
         if extra_buttons_waiting_list: app.add_extra_button()
+        app.wordsExist = 1
         if tkMessageBox.askyesno(app.master.title(),unicode(self.shortDescription,"utf-8")+which_collection+"\n"+localise("Do you want to start learning immediately?")): app.makelesson()
 
 extra_buttons_waiting_list = []
@@ -412,8 +474,8 @@ def focusButton(button):
             try: button.config(state=state)
             except: pass # maybe not a button
         for t in range(250,1000,250): # (NB avoid epilepsy's 5-30Hz!)
-          app.after(t,lambda *args:flashButton(button,"active"))
-          app.after(t+150,lambda *args:flashButton(button,"normal"))
+          app.after(t,lambda e=None,flashButton=flashButton,button=button:flashButton(button,"active"))
+          app.after(t+150,lambda e=None,flashButton=flashButton,button=button:flashButton(button,"normal"))
         # (Don't like flashing, but can't make it permanently active as it won't change when the focus does)
 
 if WMstandard: GUI_omit_statusline = 1 # unlikely to be room (and can disrupt nav)
@@ -424,16 +486,24 @@ def startTk():
             Tkinter.Frame.__init__(self, master)
             class EmptyClass: pass
             self.todo = EmptyClass() ; self.toRestore = []
+            self.ScrollUpHandler = self.ScrollDownHandler = lambda e=None:True
             global app ; app = self
             make_extra_buttons_waiting_list()
             if olpc: self.master.option_add('*font',cond(extra_buttons_waiting_list,'Helvetica 9','Helvetica 14'))
-            elif macsound and Tkinter.TkVersion>=8.6: self.master.option_add('*font','System 13') # ok with magnification.  Note >13 causes square buttons.  (Including this line causes "Big print" to work)
+            elif macsound:
+                if Tkinter.TkVersion>=8.6: self.master.option_add('*font','System 13') # ok with magnification.  Note >13 causes square buttons.  (Including this line causes "Big print" to work)
+                if "AppTranslocation" in os.getcwd(): self.todo.alert="Your Mac is using \"app translocation\" to stop Gradint from writing to its folder. This will cause many problems. Quit Gradint, drag it to a different folder and run it again."
             elif WMstandard: self.master.option_add('*font','Helvetica 7') # TODO on ALL WMstandard devices?
             if winsound or cygwin or macsound: self.master.resizable(1,0) # resizable in X direction but not Y (latter doesn't make sense, see below).  (Don't do this on X11 because on some distros it results in loss of automatic expansion as we pack more widgets.)
+            elif unix:
+                if getoutput("xlsatoms|grep COMPIZ_WINDOW").find("COMPIZ")>=0: # (not _COMPIZ_WM_WINDOW_BLUR, that's sometimes present outside Compiz)
+                  # Compiz sometimes has trouble auto-resizing our window (e.g. on Ubuntu 11.10)
+                  self.master.geometry("%dx%d" % (self.winfo_screenwidth(),self.winfo_screenheight()))
+                  if not GUI_always_big_print: self.todo.alert = "Gradint had to maximize itself because your window manager is Compiz which sometimes has trouble handling Tkinter window sizes"
             self.extra_button_callables = []
             self.pack(fill=Tkinter.BOTH,expand=1)
             self.leftPanel = Tkinter.Frame(self)
-            self.leftPanel.pack(side="left",fill=Tkinter.X,expand=1) # "fill" needed so listbox can fill later
+            self.leftPanel.pack(side="left",fill=Tkinter.BOTH,expand=1)
             self.rightPanel = None # for now
             self.cancelling = 0 # guard against multiple presses of Cancel
             self.Label = Tkinter.Label(self.leftPanel,text="Please wait a moment")
@@ -442,22 +512,40 @@ def startTk():
             # See if we can figure out what Tk is doing with the fonts (on systems without magnification):
             try:
                 f=str(self.Label.cget("font")).split()
-                nominalSize = int(f[-1]) # IF it's in that format (e.g. Windows)
-                fontRest=" ".join(f[:-1])
+                nominalSize = intor0(f[-1])
+                if nominalSize: f=" ".join(f[:-1])+" %d"
+                else: # Tk 8.5+ ?
+                    for f2 in ['set font [font actual '+' '.join(f)+']', # Tk 8.5
+                               'set font [font actual default]']: # Tk 8.6
+                      f=str(self.tk.eval(f2)).split()
+                      upNext = 0
+                      for i in range(len(f)):
+                        if f[i]=="-size": upNext=1
+                        elif upNext:
+                            nominalSize=intor0(f[i])
+                            if nominalSize<0: nominalSize,f[i] = -nominalSize,"-%d"
+                            else: f[i]="%d"
+                            break
+                      if nominalSize==long(32768)*long(65536): nominalSize = 0 # e.g. Tk 8.6 on Ubuntu 16.04 when using the first eval string above
+                      elif f2=='set font [font actual default]': nominalSize *= 0.77 # kludge for Tk 8.6 on Ubuntu 16.04 to make large-print calculation below work
+                      if nominalSize: break
+                    f=" ".join(f)
+                    if (not checkIn("%d",f)) or not nominalSize: raise Exception("wrong format") # caught below
                 pixelSize = self.Label.winfo_reqheight()-2*int(str(self.Label["borderwidth"]))-2*int(str(self.Label["pady"]))
                 # NB DO NOT try to tell Tk a desired pixel size - you may get a *larger* pixel size.  Need to work out the desired nominal size.
                 approx_lines_per_screen_when_large = 25 # TODO really? (24 at 800x600 192dpi 15in but misses the status line, but OK for advanced users.  setting 25 gives nominal 7 which is rather smaller.)
                 largeNominalSize = int(nominalSize*self.Label.winfo_screenheight()/approx_lines_per_screen_when_large/pixelSize)
                 if largeNominalSize >= nominalSize+3:
-                    self.bigPrintFont = fontRest+" "+str(largeNominalSize)
+                    self.bigPrintFont = f % largeNominalSize
+                    self.bigPrintMult = largeNominalSize*1.0/nominalSize
                     if GUI_always_big_print:
-                        self.master.option_add('*font',self.bigPrintFont)
-                        del self.bigPrintFont ; self.isBigPrint=1
+                        self.bigPrint0()
                 else: self.after(100,self.check_window_position) # (needs to happen when window is already drawn if you want it to preserve the X co-ordinate)
             except: pass # wrong font format or something - can't do it
             if winCEsound and ask_teacherMode: self.Label["font"]="Helvetica 16" # might make it slightly easier
             self.remake_cancel_button(localise("Cancel lesson"))
-            self.copyright_string = u"This is "+(u""+program_name).replace("(c)",u"\n\u00a9").replace("-",u"\u2013")
+            self.Cancel.focus() # (default focus if we don't add anything else, e.g. reader)
+            self.copyright_string = u"This is "+ensure_unicode(program_name).replace("(c)",u"\n\u00a9").replace("-",u"\u2013")
             self.Version = Tkinter.Label(self.leftPanel,text=self.copyright_string)
             addStatus(self.Version,self.copyright_string)
             if olpc: self.Version["font"]='Helvetica 9'
@@ -481,24 +569,32 @@ def startTk():
             self.Cancel = addButton(self.CancelRow,text,self.cancel,{"side":"left"})
             self.CancelRow.pack()
         def set_statusline(self,text): # ONLY from callbacks
-            if not hasattr(self,"ListBox"): return # status changes on main screen can cause too much jumping
+            if GUI_omit_statusline or not hasattr(self,"ListBox"): return # status changes on main screen can cause too much jumping
             if not "\n" in text: text += "\n(TODO: Make that a 2-line message)" # being 2 lines helps to reduce flashing problems.  but don't want to leave 2nd line blank.
             self.Version["text"] = text
+            if not winCEsound: self.balance_statusline,self.pollInterval = self.pollInterval,10
         def restore_statusline(self,*args): # ONLY from callbacks
             if not hasattr(self,"ListBox"): return
             # self.Version["text"] = self.copyright_string
             self.Version["text"] = "\n"
         def restore_copyright(self,*args): self.Version["text"] = self.copyright_string
         def addOrTestScreen_poll(self):
+            if hasattr(self,"balance_statusline"): # try to prevent flashing on some systems/languages due to long statusline causing window resize which then takes the mouse out of the button that set the long statusline etc
+                if self.Version.winfo_reqwidth() > self.ListBox.winfo_reqwidth(): self.ListBox["width"] = int(self.ListBox["width"])+1
+                else:
+                    self.pollInterval = self.balance_statusline
+                    del self.balance_statusline
             self.sync_listbox_etc()
             if self.ListBox.curselection():
                 if not self.change_button_shown:
                     self.ChangeButton.pack()
                     self.change_button_shown = 1
+                    self.Cancel["text"] = localise("Cancel selection")
             else:
                 if self.change_button_shown:
                     self.ChangeButton.pack_forget()
                     self.change_button_shown = 0
+                    self.lastText1 = 1 # force update
             if self.toRestore:
                 if not hasattr(self,"restoreButton"): self.restoreButton = addButton(self.TestEtcCol,localise("Restore"),self.restoreText,status="This button will undo\nGradint's transliteration of the input")
             elif hasattr(self,"restoreButton"):
@@ -513,14 +609,20 @@ def startTk():
             except: pass # (if the Tk for some reason doesn't support them then that's OK)
         def poll(self):
           try:
+            global voiceOption
             if hasattr(self,"ListBox"): self.addOrTestScreen_poll()
             if hasattr(self,"scriptVariant"):
               v = self.scriptVariant.get()
               if v: v=int(v)
               else: v=0
               if not v==scriptVariants.get(firstLanguage,0): self.setVariant(v)
+            if hasattr(self,"voiceOption") and not self.voiceOption.get()==voiceOption:
+              voiceOption=self.voiceOption.get() ; updateSettingsFile(settingsFile,{"voiceOption":voiceOption})
             if hasattr(self,"outputTo"):
-              outTo = self.outputTo.get()
+             outTo = self.outputTo.get()
+             if hasattr(self,"lastOutTo") and self.lastOutTo==outTo: pass
+             else:
+              self.lastOutTo = outTo
               if outTo=="0": outTo=""
               if hasattr(self,"TestTextButton"):
                 if outTo: self.TestTextButton["text"]=localise("To")+" "+outTo.upper()
@@ -542,8 +644,7 @@ def startTk():
             if hasattr(self.todo,"set_main_menu") and not recorderMode:
                 # set up the main menu (better do it on this thread just in case)
                 self.cancelling = 0 # in case just pressed "stop lesson" on a repeat - make sure Quit button will now work
-                if need1adayMessage: self.Label["text"]="If you quit before making today's lesson,\nGradint will come back in 1 hour." # "\n(This is to encourage you to learn every day.)" - no, maybe let users figure that out for themselves otherwise they might think they're being patronised
-                else: self.Label.pack_forget()
+                self.Label.pack_forget()
                 self.CancelRow.pack_forget()
                 if self.todo.set_main_menu=="keep-outrow":
                     if hasattr(self,"OutputRow"): self.OutputRow.pack(fill=Tkinter.X,expand=1) # just done pack_forget in thindown
@@ -551,7 +652,7 @@ def startTk():
                     if hasattr(self,"OutputRow"): self.OutputRow.pack_forget()
                     outRow = make_output_row(self.leftPanel)
                     if outRow: self.OutputRow=outRow
-                self.TestButton = addButton(self.leftPanel,localise("Manage word list"),self.showtest) # used to be called "Add or test words", but "Manage word list" may be better for beginners
+                self.TestButton = addButton(self.leftPanel,localise(cond(self.wordsExist,"Manage word list","Create word list")),self.showtest) # used to be called "Add or test words", but "Manage word list" may be better for beginners.  And it seems that "Create word list" is even better for absolute beginners, although it shouldn't matter if self.wordsExist is not always set back to 0 when it should be.
                 self.make_lesson_row()
                 if userNameFile:
                     global GUI_usersRow
@@ -560,7 +661,9 @@ def startTk():
                     updateUserRow(1)
                 if hasattr(self,"bigPrintFont"):
                     self.BigPrintButton = addButton(self.leftPanel,localise("Big print"),self.bigPrint)
-                    self.BigPrintButton["font"]=self.bigPrintFont
+                    try: self.BigPrintButton["font"]=self.bigPrintFont
+                    except:
+                        self.BigPrintButton.pack_forget() ; del self.BigPrintButton, self.bigPrintFont
                 self.remake_cancel_button(localise("Quit"))
                 if not GUI_omit_statusline: self.Version.pack(fill=Tkinter.X,expand=1)
                 if olpc or self.todo.set_main_menu=="test" or GUI_for_editing_only: self.showtest() # olpc: otherwise will just get a couple of options at the top and a lot of blank space (no way to centre it)
@@ -569,7 +672,7 @@ def startTk():
                 self.restore_copyright()
             if hasattr(self.todo,"alert"):
                 # we have to do it on THIS thread (especially on Windows / Cygwin; Mac OS and Linux might get away with doing it from another thread)
-                tkMessageBox.showinfo(self.master.title(),self.todo.alert)
+                tkMessageBox.showinfo(self.master.title(),S(self.todo.alert))
                 del self.todo.alert
             if hasattr(self.todo,"question"):
                 self.answer_given = tkMessageBox.askyesno(self.master.title(),self.todo.question)
@@ -627,12 +730,14 @@ def startTk():
             addStatus(entry,"Limits the maximum number of NEW words\nthat are put in each lesson")
             self.NumWords.set(words)
             addLabel(self.LessonRow,localise(cond(fileExists(progressFile),"new ","")+"words in"))
-            self.Minutes,entry = addTextBox(self.LessonRow)
-            addStatus(entry,"Limits the maximum time\nthat a lesson is allowed to take")
-            entry["width"]=3
+            self.Minutes,self.MinsEntry = addTextBox(self.LessonRow)
+            addStatus(self.MinsEntry,"Limits the maximum time\nthat a lesson is allowed to take")
+            self.MinsEntry["width"]=3
             self.Minutes.set(mins)
             addLabel(self.LessonRow,localise("mins"))
             self.MakeLessonButton=addButton(self.LessonRow,localise("Start lesson"),self.makelesson,{"side":"left"},status="Press to create customized lessons\nusing the words in your collection")
+            self.lastOutTo=-1 # so it updates the Start Lesson button if needed
+            self.MakeLessonButton.bind('<FocusIn>',(lambda e=None,app=app:app.after(10,lambda e=None,app=app:app.MinsEntry.selection_clear())))
         def sync_listbox_etc(self):
             if not hasattr(self,"vocabList"):
                 if hasattr(self,"needVocablist"): return # already waiting for main thread to make one
@@ -646,10 +751,12 @@ def startTk():
                 self.lastText1=1 # so continues below
             text1,text2 = asUnicode(self.Text1.get()),asUnicode(self.Text2.get())
             if text1==self.lastText1 and text2==self.lastText2: return
+            self.lastText1,self.lastText2 = text1,text2
+            if WMstandard and text1=="(Push OK to type A-Z)": text1=""
             for control,current,restoreTo in self.toRestore:
                 if not asUnicode(control.get())==current:
                     self.toRestore = [] ; break
-            if text1 or text2: self.Cancel["text"] = localise("Clear input boxes")
+            if text1 or text2: self.Cancel["text"] = localise(cond(self.ListBox.curselection(),"Cancel selection","Clear input boxes"))
             else: self.Cancel["text"] = localise(cond(olpc or GUI_for_editing_only,"Quit","Back to main menu"))
             h = hanzi_only(text1)
             if Tk_might_display_wrong_hanzi and not self.Label1["text"].endswith(wrong_hanzi_message) and (h or hanzi_only(text2)): self.Label1["text"]+=("\n"+wrong_hanzi_message)
@@ -661,33 +768,42 @@ def startTk():
             if synthCache:
                 cacheManagementOptions = [] # (text, oldKey, newKey, oldFile, newFile)
                 for t,l in [(text1.encode('utf-8'),secondLanguage),(text2.encode('utf-8'),firstLanguage)]:
-                    k,f = synthcache_lookup("!synth:"+t+"_"+l,justQueryCache=1)
-                    if f and (partials_langname(l) in synth_partials_voices or get_synth_if_possible(l,0)): # (2nd condition is because there's no point having these buttons if there's no chance we can synth it by any method OTHER than the cache)
-                        if k in synthCache_transtbl and k[0]=="_": cacheManagementOptions.append(("Keep in "+l+" cache",k,k[1:],0,0))
-                        elif k[0]=="_": cacheManagementOptions.append(("Keep in "+l+" cache",0,0,f,f[1:]))
-                        if k in synthCache_transtbl: cacheManagementOptions.append(("Reject from "+l+" cache",k,"__rejected_"+k,0,0))
+                    k,f = synthcache_lookup(B("!synth:")+t+B("_")+B(l),justQueryCache=1)
+                    if f:
+                      if (checkIn(partials_langname(l),synth_partials_voices) or get_synth_if_possible(l,0)): # (no point having these buttons if there's no chance we can synth it by any method OTHER than the cache)
+                        if checkIn(k,synthCache_transtbl) and B(k[:1])==B("_"): cacheManagementOptions.append(("Keep in "+l+" cache",k,k[1:],0,0))
+                        elif B(k[:1])==B("_"): cacheManagementOptions.append(("Keep in "+l+" cache",0,0,f,f[1:]))
+                        if checkIn(k,synthCache_transtbl): cacheManagementOptions.append(("Reject from "+l+" cache",k,"__rejected_"+k,0,0))
                         else: cacheManagementOptions.append(("Reject from "+l+" cache",0,0,f,"__rejected_"+f))
+                    else:
+                      k,f = synthcache_lookup(B("!synth:__rejected_")+t+B("_"+l),justQueryCache=1)
+                      if not f: k,f = synthcache_lookup(B("!synth:__rejected__")+t+B("_"+l),justQueryCache=1)
+                      if f:
+                        if checkIn(k,synthCache_transtbl): cacheManagementOptions.append(("Undo "+l+" cache reject",k,k[11:],0,0))
+                        else: cacheManagementOptions.append(("Undo "+l+" cache reject",0,0,f,f[11:]))
+                      elif l==secondLanguage and mp3web and not ';' in t: cacheManagementOptions.append(("Get from "+mp3webName,0,0,0,0))
                 if not hasattr(self,"cacheManagementOptions"):
                     self.cacheManagementOptions = []
                     self.cacheManagementButtons = []
                 if not cacheManagementOptions==self.cacheManagementOptions:
-                    self.cacheManagementOptions = cacheManagementOptions
                     for b in self.cacheManagementButtons: b.pack_forget()
+                    self.cacheManagementOptions = cacheManagementOptions
                     self.cacheManagementButtons = []
                     for txt,a,b,c,d in cacheManagementOptions: self.cacheManagementButtons.append(addButton(self.TestEtcCol,txt,lambda e=self,a=a,b=b,c=c,d=d:e.doSynthcacheManagement(a,b,c,d),status="This button is for synthCache management.\nsynthCache is explained in advanced"+extsep+"txt"))
-            self.lastText1,self.lastText2 = text1,text2
             if self.ListBox.curselection():
                 if not (text1 or text2): self.ListBox.selection_clear(0,'end') # probably just added a new word while another was selected (added a variation) - clear selection to reduce confusion
                 else: return # don't try to be clever with searches when editing an existing item (the re-ordering can be confusing)
             text1,text2 = text1.lower().replace(" ",""),text2.lower().replace(" ","") # ignore case and whitespace when searching
-            l=map(lambda (x,y):x+"="+y, filter(lambda (x,y):text1 in x.lower().replace(" ","") and text2 in y.lower().replace(" ",""),self.vocabList)[-tkNumWordsToShow:])
+            l=map(lambda x:x[0]+"="+x[1], filter(lambda x,text1=text1,text2=text2:x[0].lower().replace(" ","").find(text1)>-1 and x[1].lower().replace(" ","").find(text2)>-1,self.vocabList)[-tkNumWordsToShow:])
             l.reverse() ; synchronizeListbox(self.ListBox,l) # show in reverse order, in case the bottom of the list box is off-screen
         def doSynthcacheManagement(self,oldKey,newKey,oldFname,newFname):
             # should be a quick operation - might as well do it in the GUI thread
-            if oldKey in synthCache_transtbl:
+            if (oldKey,oldFname) == (0,0): # special for mp3web
+                self.menu_response="mp3web" ; return
+            if checkIn(oldKey,synthCache_transtbl):
                 if newKey: synthCache_transtbl[newKey]=synthCache_transtbl[oldKey]
                 else: del synthCache_transtbl[oldKey]
-                open(synthCache+os.sep+transTbl,'w').write("".join([v+" "+k+"\n" for k,v in synthCache_transtbl.items()]))
+                open(synthCache+os.sep+transTbl,'wb').write(B("").join([v+B(" ")+k+B("\n") for k,v in list(synthCache_transtbl.items())]))
             if oldFname:
                 del synthCache_contents[oldFname]
                 if newFname:
@@ -699,7 +815,12 @@ def startTk():
             for control,current,restoreTo in self.toRestore:
                 if asUnicode(control.get())==current: control.set(restoreTo)
             self.toRestore = []
-        def stripText(self,*args): self.Text1.set(fix_commas(hanzi_and_punc(asUnicode(self.Text1.get()))))
+        def stripText(self,*args):
+            t = self.Text1.get()
+            u = asUnicode(t)
+            v = fix_commas(hanzi_and_punc(u))
+            if t==u: v=asUnicode(v)
+            self.Text1.set(v)
         def thin_down_for_lesson(self):
             if hasattr(self,"OutputRow"): self.OutputRow.pack_forget()
             if hasattr(self,"CopyFromButton"):
@@ -718,17 +839,25 @@ def startTk():
                     self.ChangeButton.pack_forget()
                     self.change_button_shown = 0
                 del self.ListBox # so doesn't sync lists, or assume Cancel button is a Clear button
+                if hasattr(self,"cacheManagementButtons"):
+                    for b in self.cacheManagementButtons: b.pack_forget()
+                    del self.cacheManagementButtons,self.cacheManagementOptions
                 app.master.title(appTitle)
             self.CancelRow.pack_forget() ; self.Version.pack_forget()
             self.Label.pack() ; self.CancelRow.pack()
             self.Label["text"] = "Working..." # (to be replaced by time indication on real-time, not on output-to-file)
             self.Cancel["text"] = localise("Quit")
-        def bigPrint(self,*args):
-            self.thin_down_for_lesson()
+        def bigPrint0(self):
             self.master.option_add('*font',self.bigPrintFont)
-            self.Version["font"]=self.Label["font"]=self.bigPrintFont
+            self.sbarWidth = int(16*self.bigPrintMult)
+            self.master.option_add('*Scrollbar*width',self.sbarWidth) # (works on some systems; usually ineffective on Mac)
+            self.Label["font"]=self.bigPrintFont
             del self.bigPrintFont # (TODO do we want an option to undo it?  or would that take too much of the big print real-estate.)
             self.isBigPrint=1
+        def bigPrint(self,*args):
+            self.thin_down_for_lesson()
+            self.Version["font"]=self.bigPrintFont
+            self.bigPrint0()
             if self.rightPanel: # oops, need to re-construct it
                 global extra_buttons_waiting_list
                 extra_buttons_waiting_list = []
@@ -770,7 +899,7 @@ def startTk():
             self.Cancel["text"] = localise("Cancel lesson")
             self.menu_response = "go"
         def showtest(self,*args): # Can assume main menu is shown at the moment.
-            title = localise("Manage word list")
+            title = localise(cond(self.wordsExist,"Manage word list","Create word list"))
             if hasattr(self,"userNo"):
                 try: uname = lastUserNames[intor0(self.userNo.get())]
                 except IndexError: uname="" # can happen if it's 0 but list is empty
@@ -789,7 +918,7 @@ def startTk():
             self.row4 = addRow(self.leftPanel,1)
             self.Label1,self.Text1,self.Entry1 = addLabelledBox(self.row1,True)
             self.TestEtcCol = addRow(self.row1) # effectively adding a column to the end of the row, for "Speak" and any other buttons to do with 2nd-language text (although be careful not to add too many due to tabbing)
-            self.TestTextButton = addButton(self.TestEtcCol,"",self.testText,status="Use this button to check how the\ncomputer will pronounce words before you add them") # will set text in updateLanguageLabels
+            self.TestTextButton = addButton(self.TestEtcCol,"",self.testText,status="Use this button to check how the computer\nwill pronounce words before you add them") # will set text in updateLanguageLabels
             self.Label2,self.Text2,self.Entry2 = addLabelledBox(self.row2,True)
             if not WMstandard:
               self.Entry1.bind('<Return>',self.testText)
@@ -804,18 +933,19 @@ def startTk():
             self.L2Entry.bind('<Return>',self.changeLanguages)
             for e in [self.L1Entry,self.L2Entry]: e.bind('<Button-1>',(lambda e:e.widget.after(10,lambda e=e:selectAll(e))))
             self.ChangeLanguageButton = addButton(self.row3,"",self.changeLanguages,status="Use this button to set your\nfirst and second languages") # will set text in updateLanguageLabels
+            self.ChangeLanguageButton.bind('<FocusIn>',(lambda e=None,app=app:app.after(10,lambda e=None,app=app:app.L2Entry.selection_clear())))
+            self.AddButton.bind('<FocusIn>',(lambda e=None,app=app:app.after(10,lambda e=None,app=app:app.L1Entry.selection_clear()))) # for backwards tabbing
             if GUI_omit_settings and (vocabFile==user0[1] or fileExists(vocabFile)): self.row3.pack_forget()
-            if (not olpc) or textEditorCommand or explorerCommand: # no point doing this on the XO, because no way to run editor or file browser (unless someone's installed one) and "do it yourself" is probably not helpful in that environment
+            if textEditorCommand:
                 self.RecordedWordsButton = addButton(self.row4,"",self.showRecordedWords,{"side":"left"},status="This button lets you manage recorded\n(as opposed to computer-voiced) words")
                 row4right = addRightRow(self.row4)
                 self.EditVocabButton = addButton(row4right,"",self.openVocabFile,{"side":"left"},status="This button lets you edit your\nvocab collection in "+textEditorName)
-                if not GUI_omit_settings: addButton(row4right,"advanced"+dottxt,self.openAdvancedTxt,{"side":"left"},status="This button lets you change advanced settings\nand do other technical things - beginners beware!")
+                if not GUI_omit_settings: addButton(row4right,"advanced"+dottxt,self.openAdvancedTxt,{"side":"left"},status="Press this button to change voices,\nlearn multiple languages, etc")
                 self.make_lesson_row()
-            else:
-                # can at least have Recorded Words button now we have a buillt-in manager
+            else: # no text editor, but can at least have Recorded Words button now we have a built-in manager
                 self.make_lesson_row()
                 self.RecordedWordsButton = addButton(self.LessonRow,"",self.showRecordedWords,{"side":"right"},status="This button lets you manage recorded\n(as opposed to computer-voiced) words")
-            if ((not olpc) or textEditorCommand or explorerCommand) and lastUserNames and lastUserNames[0]: self.CopyFromButton = addButton(cond(GUI_omit_settings,row4right,self.LessonRow),localise("Copy from..."),self.showCopyFrom,{"side":"left"},status="This button lets you copy recorded\nand computer-voiced words from other users")
+            if textEditorCommand and lastUserNames and lastUserNames[0]: self.CopyFromButton = addButton(cond(GUI_omit_settings,row4right,self.LessonRow),localise("Copy from..."),self.showCopyFrom,{"side":"left"},status="This button lets you copy recorded\nand computer-voiced words from other users") # TODO if not textEditorCommand then only reason why can't have this is row4right won't be defined, need to fix that (however probably don't want to bother on XO etc)
             self.remake_cancel_button(localise(cond(olpc or GUI_for_editing_only,"Quit","Back to main menu")))
             self.ChangeButton = addButton(self.CancelRow,"",self.changeItem,{"side":"left"},status="Press to alter or to delete\nthe currently-selected word in the list") ; self.ChangeButton.pack_forget() # don't display it until select a list item
             self.updateLanguageLabels()
@@ -851,8 +981,8 @@ def startTk():
             m=Tkinter.Menu(None, tearoff=0, takefocus=0)
             for i in range(len(lastUserNames)):
                 if lastUserNames[i] and not i==intor0(self.userNo.get()):
-                    if fileExists(addUserToFname(user0[1],i)): m.add_command(label=u"Copy vocab list from "+lastUserNames[i],command=(lambda i=i,*args:self.copyVocabFrom(i)))
-                    m.add_command(label=u"Copy recordings to/from "+lastUserNames[i],command=(lambda i=i,*args:self.setToOpen((addUserToFname(user0[0],i),addUserToFname(user0[0],intor0(self.userNo.get()))))))
+                    if fileExists(addUserToFname(user0[1],i)): m.add_command(label=u"Copy vocab list from "+lastUserNames[i],command=(lambda e=None,i=i,self=self:self.copyVocabFrom(i)))
+                    m.add_command(label=u"Copy recordings to/from "+lastUserNames[i],command=(lambda e=None,i=i,self=self:self.setToOpen((addUserToFname(user0[0],i),addUserToFname(user0[0],intor0(self.userNo.get()))))))
             m.tk_popup(self.CopyFromButton.winfo_rootx(),self.CopyFromButton.winfo_rooty(),entry="0")
         def setToOpen(self,toOpen): self.menu_response,self.toOpen = "samplesCopy",toOpen
         def copyVocabFrom(self,userNo):
@@ -862,11 +992,12 @@ def startTk():
             select_userNumber(intor0(self.userNo.get()),updateGUI=0)
             vCurrent = list2set(vocabLinesWithLangs())
             o=appendVocabFileInRightLanguages()
+            if not o: return # IOError
             langs = (secondLanguage,firstLanguage)
             for newLangs,line in vCopyFrom:
-                if (newLangs,line) in vCurrent: continue # already got it
-                if not newLangs==langs: o.write("SET LANGUAGES "+" ".join(list(newLangs))+"\n")
-                o.write(line+"\n")
+                if checkIn((newLangs,line),vCurrent): continue # already got it
+                if not newLangs==langs: o.write(B("SET LANGUAGES ")+B(" ").join(list(newLangs))+B("\n"))
+                o.write(B(line)+B("\n"))
                 langs = newLangs
             o.close()
             if hasattr(self,"vocabList"): del self.vocabList # re-read
@@ -881,15 +1012,15 @@ def startTk():
             global firstLanguage,secondLanguage
             firstLanguage1=asUnicode(self.L1Text.get()).encode('utf-8')
             secondLanguage1=asUnicode(self.L2Text.get()).encode('utf-8')
-            if (firstLanguage,secondLanguage) == (firstLanguage1,secondLanguage1): # they didn't change anything
+            if (B(firstLanguage),B(secondLanguage)) == (firstLanguage1,secondLanguage1): # they didn't change anything
                 langs = ESpeakSynth().describe_supported_languages()
                 msg = (localise("To change languages, edit the boxes that say '%s' and '%s', then press the '%s' button.") % (firstLanguage,secondLanguage,localise("Change languages")))+"\n\n"+localise("Recorded words may be in ANY languages, and you may choose your own abbreviations for them.  However if you want to use the computer voice for anything then please use standard abbreviations.")
                 if langs:
-                    if tkMessageBox.askyesno(self.master.title(),msg+"  "+localise("Would you like to see a list of the standard abbreviations for languages that can be computer voiced?")): self.todo.alert = localise("Languages that can be computer voiced:")+"\n"+langs
+                    if tkMessageBox.askyesno(self.master.title(),msg+"  "+localise("Would you like to see a list of the standard abbreviations for languages that can be computer voiced?")): self.todo.alert = localise("Languages with computer voices (some better than others):")+"\n"+langs
                 else: self.todo.alert = msg+"  "+localise("(Sorry, a list of these is not available on this system - check eSpeak installation.)")
                 return
-            need_redisplay = "@variants-"+firstLanguage in GUI_translations or "@variants-"+firstLanguage1 in GUI_translations # if EITHER old or new lang has variants, MUST reconstruct that row.  (TODO also do it anyway to get the "Speaker" etc updated?  but may cause unnecessary flicker if that's no big problem)
-            firstLanguage,secondLanguage = firstLanguage1,secondLanguage1
+            need_redisplay = checkIn("@variants-"+GUI_languages.get(firstLanguage,firstLanguage),GUI_translations) or checkIn("@variants-"+GUI_languages.get(S(firstLanguage1),S(firstLanguage1)),GUI_translations) # if EITHER old or new lang has variants, MUST reconstruct that row.  (TODO also do it anyway to get the "Speaker" etc updated?  but may cause unnecessary flicker if that's no big problem)
+            firstLanguage,secondLanguage = S(firstLanguage1),S(secondLanguage1)
             updateSettingsFile(settingsFile,{"firstLanguage":firstLanguage,"secondLanguage":secondLanguage})
             if need_redisplay:
                 self.thin_down_for_lesson()
@@ -899,20 +1030,16 @@ def startTk():
         def updateLanguageLabels(self):
             # TODO things like "To" and "Speaker" need updating dynamically with localise() as well, otherwise will be localised only on restart (unless the old or new lang has variants, in which case it will be repainted anyway above)
             self.Label1["text"] = (localise("Word in %s") % localise(secondLanguage))+":"
-            if winsound or mingw32 or cygwin: self.Label1["text"] += "\n(" + localise("press Control-V to paste")+")"
-            elif macsound:
-                l = localise("press Apple-V to paste")
-                if not Tk_might_display_wrong_hanzi: l=(u""+l).replace("Apple",u"\u2318") # not sure how well this works in earlier versions
-                self.Label1["text"] += "\n("+l+")"
             self.Label2["text"] = (localise("Meaning in %s") % localise(firstLanguage))+":"
             self.L1Text.set(firstLanguage)
             self.L2Text.set(secondLanguage)
             self.L1Label["text"] = localise("Your first language")+":"
             self.L2Label["text"] = localise("second")+":"
-            self.TestTextButton["text"] = localise("Speak")
+            self.TestTextButton["text"] = localise("Speak") ; self.lastOutTo=-1 # so updates to "To WAV" etc if necessary
             if hasattr(self,"userNo") and intor0(self.userNo.get()): gui_vocabFile_name="vocab file" # don't expose which user number they are because that might change
             elif len(vocabFile)>15 and os.sep in vocabFile: gui_vocabFile_name=vocabFile[vocabFile.rindex(os.sep)+1:]
             else: gui_vocabFile_name=vocabFile
+            if gui_vocabFile_name=="vocab.txt": gui_vocabFile_name=localise(gui_vocabFile_name)
             self.AddButton["text"] = localise("Add to %s") % gui_vocabFile_name
             self.ChangeLanguageButton["text"] = localise("Change languages")
             self.ChangeButton["text"] = localise("Change or delete item")
@@ -953,7 +1080,10 @@ def startTk():
             self.menu_response="add"
         def zap_newlines(self): # in case someone pastes in text that contains newlines, better not keep them when adding to vocab
             text1,text2 = asUnicode(self.Text1.get()),asUnicode(self.Text2.get())
-            t1,t2 = text1.replace("\n"," ").replace("\r","").strip(wsp), text2.replace("\n"," ").replace("\r","").strip(wsp)
+            # (also remove the simple visual markup that Wenlin sometimes adds)
+            t1,t2=text1,text2
+            for zap in ["\n","\r","<b>","</b>","<i>","</i>","<u>","</u>"]: t1,t2=t1.replace(zap,""),t2.replace(zap,"")
+            t1,t2 = wspstrip(t1),wspstrip(t2)
             if not t1==text1: self.Text1.set(t1)
             if not t2==text2: self.Text2.set(t2)
         def getEncoder(self,*args):
@@ -983,15 +1113,18 @@ def startTk():
     def appThread(appclass):
         global app ; appclass() # sets 'app' to itself on construction
         app.master.title(appTitle)
+        app.wordsExist = words_exist()
         app.mainloop()
         closeBoxPressed = not hasattr(app.todo,"exit_ASAP")
         app = 0 # (not None - see 'app==None' below)
         if closeBoxPressed:
             if emulated_interruptMain:
                 global need_to_interrupt ; need_to_interrupt = 1
+                while RM_running: time.sleep(0.1) # ensure main thread is last to exit, sometimes needed
             else: thread.interrupt_main()
     def processing_thread():
         while not app: time.sleep(0.1) # make sure started
+        # import cProfile as profile ; return profile.run('rest_of_main()',sort=2)
         rest_of_main()
     if Tkinter.TkVersion < 8.5: # we can do the processing in the main thread, so interrupt_main works
         thread.start_new_thread(appThread,(Application,))
@@ -1002,7 +1135,7 @@ def startTk():
         appThread(Application)
 
 def hanzi_only(unitext): return u"".join(filter(lambda x:0x3000<ord(x)<0xa700 or ord(x)>=0x10000, list(unitext)))
-def hanzi_and_punc(unitext): return u"".join(filter(lambda x:0x3000<ord(x)<0xa700 or ord(x)>=0x10000 or x in '.,?;":\'()[]!0123456789-', list(remove_tone_numbers(fix_compatibility(unitext)))))
+def hanzi_and_punc(unitext): return u"".join(filter(lambda x:0x3000<ord(x)<0xa700 or ord(x)>=0x10000 or x in '.,?;:\'()[]!0123456789-', list(remove_tone_numbers(fix_compatibility(unitext))))) # no " as it could be from SGML markup
 # (exclusion of 3000 in above is deliberate, otherwise get problems with hanzi spaces being taken out by fix-compat+strip hence a non-functional 'delete non-hanzi' button appears)
 def guiVocabList(parsedVocab):
     # This needs to be fast.  Have tried writing interatively rather than filter and map, and assume stuff is NOT already unicode (so just decode rather than call ensure_unicode) + now assuming no !synth: (but can still run with .txt etc)
@@ -1019,24 +1152,26 @@ def guiVocabList(parsedVocab):
         if b.endswith(fl2): b=b[:fl2Len]
         elif b.endswith(fl3): b=readText(b)
         else: continue
-        ret.append((unicode(c,"utf-8"),unicode(b,"utf-8")))
+        ret.append((ensure_unicode(c),ensure_unicode(b)))
     return ret
 def readText(l): # see utils/transliterate.py (running guiVocabList on txt files from scanSamples)
-    l = samplesDirectory+os.sep+l
-    if l in variantFiles: # oops. just read the 1st .txt variant
-        if os.sep in l: lp=(l+os.sep)[:l.rfind(os.sep)]+os.sep
-        else: lp = ""
-        varList = filter(lambda x:x.endswith(dottxt),variantFiles[l])
+    l = B(samplesDirectory)+B(os.sep)+B(l)
+    if checkIn(l,variantFiles): # oops. just read the 1st .txt variant
+        if B(os.sep) in l: lp=(l+B(os.sep))[:l.rfind(B(os.sep))]+B(os.sep)
+        else: lp = B("")
+        varList = filter(lambda x:x.endswith(B(dottxt)),variantFiles[l])
         varList.sort() # so at least it consistently returns the same one.  TODO utils/ cache-synth.py list-synth.py synth-batchconvert-helper.py all use readText() now, can we get them to cache the other variants too?
         l = lp + varList[0]
-    return u8strip(open(l,"rb").read()).strip(wsp)
+    return bwspstrip(u8strip(read(l)))
 
 def singular(number,s):
   s=localise(s)
   if firstLanguage=="en" and number==1 and s[-1]=="s": return s[:-1]
   return s
 def localise(s):
-  d = GUI_translations.get(s,{}) ; s2 = 0
+  if s=="zh-yue" or s=="zhy": k="cant"
+  else: k=s
+  d = GUI_translations.get(k,{}) ; s2 = 0
   GUIlang = GUI_languages.get(firstLanguage,firstLanguage)
   if scriptVariants.get(GUIlang,0): s2 = d.get(GUIlang+str(scriptVariants[GUIlang]+1),0)
   if not s2: s2 = d.get(GUIlang,s)
@@ -1046,7 +1181,7 @@ if winCEsound: # some things need more squashing
     del localise
     def localise(s):
         s=GUI_translations.get(s,{}).get(firstLanguage,s)
-        return {"Your first language":"1st","second":"2nd"}.get(s,s)
+        return {"Your first language":"1st","second":"2nd","Start lesson":"Start"}.get(s,s)
 
 def synchronizeListbox(listbox,masterList):
     mi=li=0 ; toDelete = []
@@ -1085,10 +1220,7 @@ if useTK:
     textEditorCommand=explorerCommand=None
     if winsound or mingw32 or cygwin:
         textEditorName="Notepad" ; textEditorWaits=1
-        # Try Notepad++ first, otherwise plain notepad
-        textEditorCommand = programFiles+os.sep+"Notepad++"+os.sep+"notepad++.exe"
-        if fileExists(textEditorCommand): textEditorCommand='"'+textEditorCommand+'" -multiInst -notabbar -nosession'
-        else: textEditorCommand="notepad"
+        textEditorCommand="notepad"
         explorerCommand="explorer"
     elif macsound:
         textEditorName="TextEdit"
@@ -1096,21 +1228,27 @@ if useTK:
         if got_program("bbedit"):
             textEditorName="bbedit"
             textEditorCommand="bbedit -w" ; textEditorWaits=1
+        elif got_program("edit"): # TextWrangler
+            textEditorName="edit"
+            textEditorCommand="edit -w" ; textEditorWaits=1
         if sys.version.startswith("2.3.5") and "DISPLAY" in os.environ: explorerCommand = None # 'open' doesn't seem to work when running from within Python in X11 on 10.4
         else: explorerCommand="open"
     elif unix:
-        if "KDE_FULL_SESSION" is os.environ and got_program("kfmclient"):
+        if "KDE_FULL_SESSION" in os.environ and got_program("kfmclient"):
             # looks like we're in a KDE session and can use the kfmclient command
             textEditorCommand=explorerCommand="kfmclient exec"
         elif not olpc and got_program("gnome-open"):
             textEditorCommand=explorerCommand="gnome-open"
+        elif got_program("nautilus"): explorerCommand="nautilus"
+        elif got_program("pcmanfm"): explorerCommand="pcmanfm" # LXDE, LXQt
+        elif got_program("pcmanfm-qt"): explorerCommand="pcmanfm-qt" # might not work as well as pcmanfm on 24.04
         elif got_program("rox"):
             # rox is available - try using that to open directories
             # (better not use it for editor as it might not be configured)
-            # (TODO if both rox and gnome-open are available, can we tell which one the user prefers?  currently using gnome-open)
+            # (TODO if both rox and gnome are available, can we tell which one the user prefers?)
             explorerCommand="rox"
         # anyway, see if we can find a nice editor
-        for editor in ["gedit","nedit","kedit","xedit"]:
+        for editor in ["leafpad","featherpad","gedit","nedit","kedit","xedit"]:
             if got_program(editor):
                 textEditorName=textEditorCommand=editor
                 textEditorWaits = 1
@@ -1120,19 +1258,21 @@ if useTK:
                 break
     # End of finding editor - now start GUI
     try:
-        import thread,Tkinter,tkMessageBox
+        try: import thread
+        except ImportError: import _thread as thread
+        try: import Tkinter,tkMessageBox
+        except:
+            import tkinter as Tkinter
+            from tkinter import messagebox as tkMessageBox
         forceRadio=(macsound and 8.49<Tkinter.TkVersion<8.59) # indicatoron doesn't do very well in OS X 10.6 (Tk 8.5) unless we patched it
         if olpc:
             def interrupt_main(): os.kill(os.getpid(),2) # sigint
             thread.interrupt_main = interrupt_main
             # (os.kill is more reliable than interrupt_main() on OLPC, *but* on Debian Sarge (2.4 kernel) threads are processes so DON'T do this.)
         elif not hasattr(thread,"interrupt_main"): emulated_interruptMain = 1
-        else:
-            try: # work around the "int object is not callable" thing on some platforms' interrupt_main
-                import signal
-                def raise_int(*args): raise KeyboardInterrupt
-                signal.signal(signal.SIGINT,raise_int)
-            except: pass
+        elif signal: # work around the "int object is not callable" thing on some platforms' interrupt_main
+            def raise_int(*args): raise KeyboardInterrupt
+            signal.signal(signal.SIGINT,raise_int)
     except RuntimeError:
         useTK = 0
         if __name__=="__main__": show_warning("Cannot start the GUI due to a Tk error")
@@ -1143,31 +1283,35 @@ if useTK:
 def openDirectory(dir,inGuiThread=0):
     if winCEsound:
         if not dir[0]=="\\": dir=os.getcwd()+cwd_addSep+dir # must be absolute
-        ctypes.cdll.coredll.ShellExecuteEx(ctypes.byref(ShellExecuteInfo(60,File=u"\\Windows\\fexplore",Parameters=u""+dir)))
+        ctypes.cdll.coredll.ShellExecuteEx(ctypes.byref(ShellExecuteInfo(60,File=u"\\Windows\\fexplore",Parameters=ensure_unicode(dir))))
     elif explorerCommand:
+        if ' ' in dir: dir='"'+dir+'"'
         cmd = explorerCommand+" "+dir
         if winsound or mingw32: cmd="start "+cmd # (not needed on XP but is on Vista)
         elif unix: cmd += "&"
         os.system(cmd)
     else:
-        msg = "Don't know how to start the file explorer.  Please open the %s directory (in %s)" % (dir,os.getcwd())
+        msg = ""
+        if not dir.startswith(os.sep): msg=" (in %s)" % os.getcwd()
+        msg = "Don't know how to start the file explorer.  Please open the %s directory%s" % (dir,msg)
         if inGuiThread: tkMessageBox.showinfo(app.master.title(),msg)
         else: waitOnMessage(msg)
 
-def sanityCheck(text,language,pauseOnError=0): # text is utf-8; returns error message if any
+def generalCheck(text,language,pauseOnError=0): # text is utf-8; returns error message if any
     if not text: return # always OK empty strings
     if pauseOnError:
-        ret = sanityCheck(text,language)
+        ret = generalCheck(text,language)
         if ret: waitOnMessage(ret)
         return ret
     if language=="zh":
-        allDigits = True
-        for t in text:
+        allDigits = True ; text=B(text)
+        for i in xrange(len(text)):
+            t = text[i:i+1]
             if ord(t)>127: return # got hanzi or tone marks
-            if t in "12345": return # got tone numbers
-            if t not in "0123456789. ": allDigits = False
+            if t in B("12345"): return # got tone numbers
+            if t not in B("0123456789. "): allDigits = False
         if allDigits: return
-        return "Pinyin needs tones.  Please go back and add tone numbers to "+text+"."+cond(startBrowser("http://www.pristine.com.tw/lexicon.php?query="+fix_pinyin(text,[]).replace("1","1 ").replace("2","2 ").replace("3","3 ").replace("4","4 ").replace("5"," ").replace("  "," ").strip(wsp).replace(" ","+"))," Gradint has pointed your web browser at an online dictionary that might help.","")
+        return B("Pinyin needs tones.  Please go back and add tone numbers to ")+text+B(".")+cond(startBrowser(B("http://www.mdbg.net/chinese/dictionary?wdqb=")+bwspstrip(fix_pinyin(text,[])).replace(B("5"),B("")).replace(B(" "),B("+"))),B(" Gradint has pointed your web browser at an online dictionary that might help."),B(""))
 
 def check_for_slacking():
     if fileExists(progressFile): checkAge(progressFile,localise("It has been %d days since your last Gradint lesson.  Please try to have one every day."))
@@ -1182,12 +1326,12 @@ def checkAge(fname,message):
     if days>=5 and (days%5)==0: waitOnMessage(message % days)
 
 def s60_addVocab():
-  label1,label2 = u""+localise("Word in %s") % localise(secondLanguage),u""+localise("Meaning in %s") % localise(firstLanguage)
+  label1,label2 = ensure_unicode(localise("Word in %s") % localise(secondLanguage)),ensure_unicode(localise("Meaning in %s") % localise(firstLanguage))
   while True:
     result = appuifw.multi_query(label1,label2) # unfortunately multi_query can't take default items (and sometimes no T9!), but Form is too awkward (can't see T9 mode + requires 2-button save via Options) and non-multi query would be even more modal
     if not result: return # cancelled
     l2,l1 = result # guaranteed to both be populated
-    while sanityCheck(l2.encode('utf-8'),secondLanguage,1):
+    while generalCheck(l2.encode('utf-8'),secondLanguage,1):
         l2=appuifw.query(label1,"text",u"")
         if not l2: return # cancelled
     # TODO detect duplicates like Tk GUI does?
@@ -1195,7 +1339,7 @@ def s60_addVocab():
     appendVocabFileInRightLanguages().write((l2+"="+l1+"\n").encode("utf-8"))
 def s60_changeLang():
     global firstLanguage,secondLanguage
-    result = appuifw.multi_query(u""+localise("Your first language")+" (e.g. "+firstLanguage+")",u""+localise("second")+" (e.g. "+secondLanguage+")")
+    result = appuifw.multi_query(ensure_unicode(localise("Your first language")+" (e.g. "+firstLanguage+")"),ensure_unicode(localise("second")+" (e.g. "+secondLanguage+")"))
     if not result: return # cancelled
     l1,l2 = result
     firstLanguage,secondLanguage = l1.encode('utf-8').lower(),l2.encode('utf-8').lower()
@@ -1209,7 +1353,7 @@ def s60_runLesson():
 def s60_viewVocab():
     global justSynthesize
     doLabel("Reading your vocab list, please wait...")
-    vList = map(lambda (l2,l1):l2+u"="+l1, guiVocabList(parseSynthVocab(vocabFile,1)))
+    vList = map(lambda x:x[0]+u"="+x[1], guiVocabList(parseSynthVocab(vocabFile,1)))
     if not vList: return waitOnMessage("Your computer-voiced vocab list is empty.")
     while True:
       appuifw.app.body = None
@@ -1219,8 +1363,8 @@ def s60_viewVocab():
       action = appuifw.popup_menu([u"Speak (just "+secondLanguage+")",u"Speak ("+secondLanguage+" and "+firstLanguage+")",u"Change "+secondLanguage,u"Change "+firstLanguage,u"Delete item",u"Cancel"], vList[sel])
       if action==0 or action==1:
         doLabel("Speaking...")
-        justSynthesize = secondLanguage+" "+l2.encode('utf-8')
-        if action==1: justSynthesize += ("#"+firstLanguage+" "+l1.encode('utf-8'))
+        justSynthesize = B(secondLanguage)+B(" ")+l2.encode('utf-8')
+        if action==1: justSynthesize += (B('#')+B(firstLanguage)+B(" ")+l1.encode('utf-8'))
         just_synthesize()
         justSynthesize = ""
       elif action==5: pass
@@ -1229,11 +1373,11 @@ def s60_viewVocab():
           oldL1,oldL2 = l1,l2
           if action==2:
               first=1
-              while first or (l2 and sanityCheck(l2.encode('utf-8'),secondLanguage,1)):
-                  first=0 ; l2=appuifw.query(u""+secondLanguage,"text",l2)
+              while first or (l2 and generalCheck(l2.encode('utf-8'),secondLanguage,1)):
+                  first=0 ; l2=appuifw.query(ensure_unicode(secondLanguage),"text",l2)
               if not l2: continue
           elif action==3:
-              l1 = appuifw.query(u""+firstLanguage,"text",l1)
+              l1 = appuifw.query(ensure_unicode(firstLanguage),"text",l1)
               if not l1: continue
           doLabel("Processing")
           delOrReplace(oldL2,oldL1,l2,l1,cond(action==4,"delete","replace"))
@@ -1241,36 +1385,81 @@ def s60_viewVocab():
               del vList[sel]
               if not vList: return # empty
           else: vList[sel] = l2+"="+l1
+def android_addVocab():
+  while True:
+    l2 = None
+    while not l2 or generalCheck(l2.encode('utf-8'),secondLanguage,1):
+      l2 = android.dialogGetInput("Add word","Word in %s" % localise(secondLanguage)).result
+      if not l2: return # cancelled
+    l1 = android.dialogGetInput("Add word","Meaning in %s" % localise(firstLanguage)).result
+    if not l1: return # cancelled
+    # TODO detect duplicates like Tk GUI does?
+    android.makeToast(u"Added "+l2+"="+l1)
+    appendVocabFileInRightLanguages().write((l2+"="+l1+"\n").encode("utf-8"))
+def android_changeLang():
+    global firstLanguage,secondLanguage
+    l1 = android.dialogGetInput("Gradint","Enter your first language",firstLanguage).result
+    if not l1: return # cancelled
+    l2 = android.dialogGetInput("Gradint","Enter your second language",secondLanguage).result
+    if not l2: return # cancelled
+    firstLanguage,secondLanguage = l1.encode('utf-8').lower(),l2.encode('utf-8').lower()
+    updateSettingsFile(settingsFile,{"firstLanguage":firstLanguage,"secondLanguage":secondLanguage})
 
 def delOrReplace(L2toDel,L1toDel,newL2,newL1,action="delete"):
     langs = [secondLanguage,firstLanguage]
-    v=u8strip(open(vocabFile,"rb").read()).replace("\r\n","\n").replace("\r","\n")
-    o=open(vocabFile,"w") ; found = 0
-    if last_u8strip_found_BOM: o.write('\xef\xbb\xbf') # re-write it
-    v=v.split("\n")
+    v=u8strip(read(vocabFile)).replace(B("\r\n"),B("\n")).replace(B("\r"),B("\n"))
+    if paranoid_file_management:
+        fname = os.tempnam()
+        o = open(fname,"w")
+    else: o=open(vocabFile,"w")
+    found = 0
+    if last_u8strip_found_BOM: writeB(o,LB('\xef\xbb\xbf')) # re-write it
+    v=v.split(B("\n"))
     if v and not v[-1]: v=v[:-1] # don't add an extra blank line at end
     for l in v:
         l2=l.lower()
-        if l2.startswith("set language ") or l2.startswith("set languages "):
-            langs=l.split()[2:] ; o.write(l+"\n") ; continue
-        thisLine=map(lambda x:x.strip(wsp),l.split("=",len(langs)-1))
+        if l2.startswith(B("set language ")) or l2.startswith(B("set languages ")):
+            langs=map(S,l.split()[2:]) ; writeB(o,l+B("\n")) ; continue
+        thisLine=map(bwspstrip,l.split(B("="),len(langs)-1))
         if (langs==[secondLanguage,firstLanguage] and thisLine==[L2toDel.encode('utf-8'),L1toDel.encode('utf-8')]) or (langs==[firstLanguage,secondLanguage] and thisLine==[L1toDel.encode('utf-8'),L2toDel.encode('utf-8')]):
             # delete this line.  and maybe replace it
             found = 1
             if action=="replace":
-                if langs==[secondLanguage,firstLanguage]: o.write(newL2.encode("utf-8")+"="+newL1.encode("utf-8")+"\n")
-                else: o.write(newL1.encode("utf-8")+"="+newL2.encode("utf-8")+"\n")
-        else: o.write(l+"\n")
+                if langs==[secondLanguage,firstLanguage]: writeB(o,newL2.encode("utf-8")+B("=")+newL1.encode("utf-8")+B("\n"))
+                else: writeB(o,newL1.encode("utf-8")+B("=")+newL2.encode("utf-8")+B("\n"))
+        else: writeB(o,l+B("\n"))
     o.close()
+    if paranoid_file_management:
+        write(vocabFile,read(fname))
+        os.remove(fname)
     return found
 
-def maybeCanSynth(lang): return lang in partials_langs or get_synth_if_possible(lang,0) or synthCache
+def maybeCanSynth(lang): return checkIn(lang,synth_partials_voices) or get_synth_if_possible(lang,0) or synthCache
+def android_main_menu():
+  while True:
+    menu=[]
+    if maybeCanSynth(secondLanguage):
+        menu.append((unicode(localise("Just speak a word")),primitive_synthloop))
+        doVocab = maybeCanSynth(firstLanguage)
+        if doVocab: menu.append((unicode(localise("Add word to my vocab")),android_addVocab))
+        menu.append((unicode(localise("Make lesson from vocab")),lesson_loop))
+        # if doVocab: menu.append((u"View/change vocab",android_viewVocab)) # (TODO but lower priority because SL4A has an editor)
+    else: menu.append((unicode(localise("Make lesson")),lesson_loop))
+    menu += [(unicode(localise("Record word(s) with mic")),android_recordWord),(unicode(localise("Change languages")),android_changeLang)]
+    menu.append((unicode(localise("Quit")),None))
+    android.dialogCreateAlert("Gradint","Choose an action")
+    android.dialogSetItems(map (lambda x:x[0], menu))
+    android.dialogShow()
+    try: function = menu[android.dialogGetResponse().result['item']][1]
+    except KeyError: break # probably an error condition: don't try to redisplay, just quit
+    if function: function() # and redisplay after
+    else: break # quit
 def s60_main_menu():
   while True:
     appuifw.app.body = None # NOT text saying version no etc - has distracting blinking cursor
     menu=[]
     if maybeCanSynth(secondLanguage):
-        menu.append((u"Just speak a word",primitive_synthloop))
+        menu.append((u"Just speak a word",primitive_synthloop)) # no localise() as S60 is not guaranteed to be able to display the characters
         doVocab = maybeCanSynth(firstLanguage)
         if doVocab: menu.append((u"Add word to my vocab",s60_addVocab))
         menu.append((u"Make lesson from vocab",s60_runLesson))
@@ -1286,20 +1475,18 @@ def s60_main_menu():
 
 def downloadLAME():
     # Sourceforge keep making this harder!
-    return not system("""if which curl >/dev/null 2>/dev/null; then export Curl="curl -L"; else export Curl="wget -O -"; fi
-if ! test -e lame*.tar.gz; then
-  export Link="$($Curl "http://sourceforge.net/project/showfiles.php?group_id=290&package_id=309"|grep tar.gz|head -1)"
-  echo "Got HTML: $Link" 1>&2
-  export Link="$(echo "$Link"|sed -e 's,href="/,href="http://sourceforge.net/,' -e 's/.*http:/http:/' -e 's/.tar.gz.*/.tar.gz/')"
-  echo "Following link to $Link" 1>&2
-  if ! $Curl "$Link" > lame.tar.gz; then
+    # Removed code to check for latest version, as we
+    # can't use v3.100 due to Lame bug 488.
+    return not system("""if which curl >/dev/null 2>/dev/null; then Curl="curl -L"; else Curl="wget -O -"; fi
+if ! [ -e lame*.tar.gz ]; then
+  if ! $Curl "https://sourceforge.net/projects/lame/files/lame/3.99/lame-3.99.5.tar.gz/download" > lame.tar.gz; then
     rm -f lame.tar.gz; exit 1
   fi
   if grep downloads.sourceforge lame.tar.gz 2>/dev/null; then
-    export Link="$(cat lame.tar.gz|grep downloads.sourceforge|head -1)"
-    echo "Got HTML 2: $Link" 1>&2
-    export Link="$(echo "$Link"|sed -e 's/.*http/http/' -e 's,.*/projects,http://sourceforge.net/projects,' -e 's/".*//')"
-    echo "Following link 2 to $Link" 1>&2
+    Link="$(cat lame.tar.gz|grep downloads.sourceforge|head -1)"
+    echo "Got HTML: $Link" >&2
+    Link="$(echo "$Link"|sed -e 's/.*http/http/' -e 's,.*/projects,http://sourceforge.net/projects,' -e 's/".*//')"
+    echo "Following link to $Link" >&2
     if ! $Curl "$Link" > lame.tar.gz; then
       rm -f lame.tar.gz; exit 1
     fi
@@ -1308,7 +1495,14 @@ fi""")
 
 def gui_event_loop():
     app.todo.set_main_menu = 1 ; braveUser = 0
-    if (orig_onceperday&2) and not need1adayMessage: check_for_slacking() # when running every day + 1st run of today
+    global disable_once_per_day
+    if disable_once_per_day==2:
+      disable_once_per_day = cond(getYN(localise("Do you want Gradint to start by itself and remind you to practise?")),0,1)
+      updateSettingsFile("advanced"+dottxt,{"disable_once_per_day":disable_once_per_day})
+      if disable_once_per_day: # signal the background process to stop next time
+        try: os.remove("background"+dottxt)
+        except: pass
+    if orig_onceperday&2: check_for_slacking()
     while app:
         while not hasattr(app,"menu_response"):
             if warnings_printed: waitOnMessage("") # If running gui_event_loop, better put any warnings in a separate dialogue now, rather than waiting for user to get one via 'make lesson' or some other method
@@ -1320,12 +1514,16 @@ def gui_event_loop():
             if emulated_interruptMain: check_for_interrupts()
             time.sleep(0.3)
         menu_response = app.menu_response
+        del app.menu_response
         if menu_response=="input": # WMstandard
             app.todo.input_response=raw_input()
         elif menu_response=="go":
             gui_outputTo_start()
             if not soundCollector: app.todo.add_briefinterrupt_button = 1
             try: lesson_loop()
+            except PromptException:
+                prEx = sys.exc_info()[1]
+                waitOnMessage("Problem finding prompts:\n"+prEx.message) # and don't quit, user may be able to fix
             except KeyboardInterrupt: pass # probably pressed Cancel Lesson while it was still being made (i.e. before handleInterrupt)
             if app and not soundCollector: app.todo.remove_briefinterrupt_button = 1 # (not app if it's closed by the close box)
             gui_outputTo_end()
@@ -1334,21 +1532,21 @@ def gui_event_loop():
         elif menu_response=="edit":
             if not braveUser and fileExists(vocabFile) and open(vocabFile).readline().find("# This is vocab.txt.")==-1: braveUser=1
             if winCEsound:
-                if braveUser or getYN("You can break things if you don't read what it says and keep to the same format.  Continue?"):
+                if braveUser or getYN("You must read what it says and keep to the same format.  Continue?"):
                     braveUser = 1
                     # WinCE Word does not save non-Western characters when saving plain text (even if there's a Unicode "cookie")
                     waitOnMessage("WARNING: Word may not save non-Western characters properly.  Try an editor like MADE instead (need to set its font).") # TODO Flinkware MADE version 2.0.0 has been known to insert spurious carriage returns at occasional points in large text files
                     if not app.fileToEdit[0]=="\\": app.fileToEdit=os.getcwd()+cwd_addSep+app.fileToEdit # must be absolute
                     if not fileExists(app.fileToEdit): open(app.fileToEdit,"w") # at least make sure it exists
-                    ctypes.cdll.coredll.ShellExecuteEx(ctypes.byref(ShellExecuteInfo(60,File=u""+app.fileToEdit)))
+                    ctypes.cdll.coredll.ShellExecuteEx(ctypes.byref(ShellExecuteInfo(60,File=ensure_unicode(app.fileToEdit))))
                     waitOnMessage("When you've finished editing "+app.fileToEdit+", close it and start gradint again.")
                     return
             elif textEditorCommand:
-                if braveUser or getYN("About to open "+app.fileToEdit+" in "+textEditorName+".\nYou can break things if you don't read what it says and keep to the same format.\nDo you really want to continue?"):
+                if braveUser or getYN("Open "+app.fileToEdit+" in "+textEditorName+"?\n(You must read what it says and keep to the same format.)"):
                     braveUser = 1 ; fileToEdit=app.fileToEdit
                     if not fileExists(fileToEdit): open(fileToEdit,"w") # at least make sure it exists
                     if textEditorWaits:
-                        oldContents = open(fileToEdit,"rb").read()
+                        oldContents = read(fileToEdit)
                         if paranoid_file_management: # run the editor on a temp file instead (e.g. because gedit can fail when saving over ftpfs)
                             fileToEdit=os.tempnam()+dottxt
                             open(fileToEdit,"w").write(oldContents)
@@ -1359,10 +1557,11 @@ def gui_event_loop():
                         t = time.time()
                         system(cmd)
                         if time.time() < t+3: waitOnMessage(textEditorName+" returned control to Gradint in less than 3 seconds.  Perhaps you already had an instance running and it loaded the file remotely.  Press OK when you have finished editing the file.")
-                        newContents = open(fileToEdit,"rb").read()
+                        newContents = read(fileToEdit)
                         if not newContents==oldContents:
-                            if paranoid_file_management: open(app.fileToEdit,"w").write(newContents)
-                            if app.fileToEdit==vocabFile: del app.vocabList # re-read
+                            if paranoid_file_management: write(app.fileToEdit,newContents)
+                            if app.fileToEdit==vocabFile:
+                                app.wordsExist=1 ; del app.vocabList # re-read
                             else: waitOnMessage("The changes you made to "+app.fileToEdit+" will take effect when you quit Gradint and start it again.")
                         del oldContents,newContents
                         if paranoid_file_management: os.remove(fileToEdit) # the temp file
@@ -1387,40 +1586,84 @@ def gui_event_loop():
             text1 = asUnicode(app.Text1.get()).encode('utf-8') ; text2 = asUnicode(app.Text2.get()).encode('utf-8')
             if not text1 and not text2: app.todo.alert=u"Before pressing the "+localise("Speak")+u" button, you need to type the text you want to hear into the box."
             else:
-              msg=sanityCheck(text1,secondLanguage)
-              if msg: app.todo.alert=u""+msg
+              if text1.startswith(B('#')): msg="" # see below
+              else: msg=generalCheck(text1,secondLanguage)
+              if msg: app.todo.alert=ensure_unicode(msg)
               else:
                 app.set_watch_cursor = 1 ; app.toRestore = []
                 global justSynthesize ; justSynthesize = ""
                 def doControl(text,lang,control):
-                    global justSynthesize
+                    global justSynthesize ; text=B(text)
                     restoreTo = asUnicode(control.get())
-                    if text:
-                        if can_be_synthesized("!synth:"+text+"_"+lang): justSynthesize += ("#"+lang+" "+text)
-                        else: app.todo.alert="Cannot find a synthesizer that can say '"+text+"' in language '"+lang+"' on this system"
-                        t=transliterates_differently(text,lang)
-                        if t:
+                    if text.startswith(B('#')): justSynthesize = B(justSynthesize)+text # hack for direct control of just_synthesize from the GUI (TODO document it in advanced.txt? NB we also bypass the GUI transliteration in the block below)
+                    elif text:
+                        if can_be_synthesized(B("!synth:")+text+B("_")+B(lang)):
+                            justSynthesize=B(justSynthesize)+(B("#")+B(lang)+B(" ")+B(text))
+                        else: app.todo.alert=B("Cannot find a synthesizer that can say '")+text+B("' in language '")+B(lang)+B("' on this system")
+                        t=S(transliterates_differently(text,lang))
+                        if t: # (don't go straight into len() stuff, it could be None)
+                          if unix and len(t)>300 and hasattr(app,"isBigPrint"): app.todo.alert="Transliteration suppressed to work around Ubuntu bug 731424" # https://bugs.launchpad.net/ubuntu/+bug/731424
+                          else:
                             control.set(t) ; app.toRestore.append((control,t,restoreTo))
                 doControl(text1,secondLanguage,app.Text1)
-                def doSynth():
-                    gui_outputTo_start() ; just_synthesize() ; gui_outputTo_end()
+                def doSynth(openDir=True):
+                    gui_outputTo_start() ; just_synthesize() ; gui_outputTo_end(openDir)
                     global justSynthesize ; justSynthesize = ""
                     if app: app.unset_watch_cursor = 1 # otherwise was closed by the close box
-                if ask_teacherMode and text1 and text2: # Do the L2, then ask if actually WANT the L1 as well (might be useful on WinCE etc, search-and-demonstrate-L2)
-                    doSynth()
-                    if app and getYN("Also speak the %s?" % firstLanguage):
-                        doControl(text2,firstLanguage,app.Text2)
-                        doSynth()
-                else:
-                    doControl(text2,firstLanguage,app.Text2)
+                if text1 and text2:
+                  if app and hasattr(app,"outputTo") and app.outputTo.get() and not app.outputTo.get()=="0":
+                    if getYN("Save %s and %s to separate files?" % (secondLanguage,firstLanguage)): doSynth(False)
+                  elif ask_teacherMode: # Do the L2, then ask if actually WANT the L1 as well (might be useful on WinCE etc, search-and-demonstrate-L2)
                     doSynth()
+                    if app and not getYN("Also speak the %s?" % firstLanguage):
+                      continue
+                doControl(text2,firstLanguage,app.Text2)
+                doSynth()
+        elif menu_response=="mp3web":
+          url=[] ; text1 = asUnicode(app.Text1.get())
+          for c in list(text1.encode("utf-8")):
+            if ord(',')<=ord(c)<=ord('9') or ord('a')<=ord(c.lower())<=ord('z'): url.append(c)
+            else: url.append("%"+hex(ord(c))[2:])
+          def scanDirs():
+           dd={} ; found=0
+           for d in downloadsDirs:
+            if isDirectory(d):
+             found=1
+             for f in os.listdir(d): dd[d+os.sep+f]=1
+           return dd,found
+          oldLs,found = scanDirs()
+          if downloadsDirs and not found: app.todo.alert=localise("Please set downloadsDirs in advanced"+dottxt)
+          elif not url: app.todo.alert=localise("You need to type a word in the box before you can press this button")
+          elif not startBrowser(mp3web.replace("$Word","".join(url)).replace("$Lang",secondLanguage)): app.todo.alert = localise("Can't start the web browser")
+          elif downloadsDirs:
+            waitOnMessage(localise("If the word is there, download it. When you press OK, Gradint will check for downloads."))
+            if not app: break
+            found=0
+            for f in scanDirs()[0].keys():
+              if not checkIn(f,oldLs) and (f.lower().endswith(dotmp3) or f.lower().endswith(dotwav)) and getYN("Use "+f[f.rfind(os.sep)+1:]+"?"): # TODO don't ask this question too many times if there are many and they're all 'no'
+                system("mp3gain -r -s r -k -d 10 \""+f+"\"") # (if mp3gain command is available; ignore errors if not (TODO document in advanced.txt)) (note: doing here not after the move, in case synthCache is over ftpfs mount or something)
+                uf=scFile=text1.encode("utf-8")+"_"+secondLanguage+f[-4:].lower()
+                try:
+                  if winCEsound: raise IOError
+                  else: o=open(synthCache+os.sep+scFile,"wb")
+                except IOError:
+                  uf=unicode2filename(text1+"_"+secondLanguage+f[-4:].lower())
+                  o=open(synthCache+os.sep+uf,"wb")
+                  synthCache_transtbl[scFile]=uf
+                  open(synthCache+os.sep+transTbl,'a').write(uf+" "+scFile+"\n")
+                synthCache_contents[uf]=1
+                o.write(open(f,"rb").read()) ; o.close() ; os.remove(f)
+                app.lastText1 = 1 # ensure different
+                found=1 ; break
+            if not found: app.todo.alert="No new sounds found"
         elif menu_response=="get-encoder":
           if winsound or mingw32:
-            if getYN("Gradint can use Windows Media Encoder to make WMA files, which can be played on most pocket MP3 players and mobiles etc.  Do you want to go to the Microsoft site to install Windows Media Encoder now?"):
-              if not startBrowser('http://www.microsoft.com/windows/windowsmedia/forpros/encoder/default.mspx'): app.todo.alert = "There was a problem starting the web browser.  Please install manually (see notes in advanced.txt)."
-              else:
-                app.setLabel("Waiting for you to install Media Encoder")
-                while not fileExists(programFiles+"\\Windows Media Components\\Encoder\\WMCmd.vbs"): time.sleep(1)
+            assert 0, "Windows Media Encoder no longer available for new installations"
+            #if getYN("Gradint can use Windows Media Encoder to make WMA files, which can be played on most pocket MP3 players and mobiles etc.  Do you want to go to the Microsoft site to install Windows Media Encoder now?"):
+            #  if not startBrowser('http://www.microsoft.com/windows/windowsmedia/forpros/encoder/default.mspx'): app.todo.alert = "There was a problem starting the web browser.  Please install manually (see notes in advanced.txt)."
+            #  else:
+            #    app.setLabel("Waiting for you to install Media Encoder")
+            #    while not fileExists(programFiles+"\\Windows Media Components\\Encoder\\WMCmd.vbs"): time.sleep(1)
           else:
             if getYN("Do you really want to download and compile the LAME MP3 encoder? (this may take a while)"):
               app.setLabel("Downloading...") ; worked=0
@@ -1430,30 +1673,29 @@ def gui_event_loop():
                 if not getYN("Download failed.  Try again?"): break
               if worked:
                 app.setLabel("Compiling...")
-                if system("""tar -zxvf lame*.tar.gz && cd lame-* && if ./configure && make; then ln -s $(pwd)/frontend/lame ../lame || true; else cd .. ; rm -rf lame*; exit 1; fi"""): app.todo.alert = "Compile failed"
+                if system("""tar -zxvf lame*.tar.gz && cd lame-* && if ./configure && make; then ln -s $(pwd)/frontend/lame ../lame || true; else cd .. ; rm -rf lame*; exit 1; fi"""):
+                    app.todo.alert = "Compile failed"
+                    if macsound:
+                        app.todo.alert += ". Check the system has Xcode with command-line license accepted (try running gcc from the Terminal)"
+                        # might be asked to run: sudo xcodebuild -license
           app.todo.set_main_menu = 1
         elif (menu_response=="add" or menu_response=="replace") and not (app.Text1.get() and app.Text2.get()): app.todo.alert="You need to type text in both boxes before adding the word/meaning pair to "+vocabFile
-        elif menu_response=="add" and hasattr(app,"vocabList") and (asUnicode(app.Text1.get()),asUnicode(app.Text2.get())) in app.vocabList:
+        elif menu_response=="add" and hasattr(app,"vocabList") and checkIn((asUnicode(app.Text1.get()),asUnicode(app.Text2.get())),app.vocabList):
             # Trying to add a word that's already there - do we interpret this as a progress adjustment?
             app.set_watch_cursor = 1
             t1,t2 = asUnicode(app.Text1.get()),asUnicode(app.Text2.get())
             lang2,lang1=t1.lower(),t2.lower() # because it's .lower()'d in progress.txt
             d = ProgressDatabase(0)
-            l1find = "!synth:"+lang1.encode('utf-8')+"_"+firstLanguage
+            l1find = S(B("!synth:")+lang1.encode('utf-8')+B("_"+firstLanguage))
             found = 0
-            msg=(u""+localise("%s=%s is already in %s.")) % (t1,t2,vocabFile)
+            msg=(ensure_unicode(localise("%s=%s is already in %s.")) % (t1,t2,vocabFile))
             for listToCheck in [d.data,d.unavail]:
               if found: break
               for item in listToCheck:
-                if (item[1]==l1find or (type(item[1])==type([]) and l1find in item[1])) and item[2]=="!synth:"+lang2.encode('utf-8')+"_"+secondLanguage:
+                if (item[1]==l1find or (type(item[1])==type([]) and checkIn(l1find,item[1]))) and item[2]==S(B("!synth:")+lang2.encode('utf-8')+B("_"+secondLanguage)):
                     if not item[0]: break # not done yet - as not-found
+                    newItem0 = reviseCount(item[0])
                     app.unset_watch_cursor = 1
-                    # suggested reduction:
-                    thresholds=[1,2,knownThreshold,reallyKnownThreshold,meaningTestThreshold,randomDropThreshold,randomDropThreshold2] ; thresholds.sort() ; thresholds.reverse()
-                    newItem0 = 0
-                    for i in range(len(thresholds)-1):
-                        if item[0]>thresholds[i]:
-                            newItem0=thresholds[i+1] ; break
                     if getYN(msg+" "+localise("Repeat count is %d. Reduce this to %d for extra revision?" % (item[0],newItem0))):
                         app.set_watch_cursor = 1
                         listToCheck.remove(item)
@@ -1466,14 +1708,17 @@ def gui_event_loop():
                 app.todo.alert=msg+" "+localise("Repeat count is 0, so we cannot reduce it for extra revision.")
         elif menu_response=="add":
             text1 = asUnicode(app.Text1.get()).encode('utf-8') ; text2 = asUnicode(app.Text2.get()).encode('utf-8')
-            msg=sanityCheck(text1,secondLanguage)
-            if msg: app.todo.alert=u""+msg
+            msg=generalCheck(text1,secondLanguage)
+            if msg: app.todo.alert=ensure_unicode(msg)
             else:
                 o=appendVocabFileInRightLanguages()
-                o.write(text1+"="+text2+"\n") # was " = " but it slows down parseSynthVocab
+                if not o: continue # IOError
+                writeB(o,text1+B("=")+text2+B("\n")) # was " = " but it slows down parseSynthVocab
                 o.close()
+                if paranoid_file_management:
+                    if filelen(vocabFile)<filelen(vocabFile+"~") or chr(0) in readB(open(vocabFile,"rb"),1024): app.todo.alert="Vocab file corruption! You'd better restore the ~ backup."
                 if hasattr(app,"vocabList"): app.vocabList.append((ensure_unicode(text1),ensure_unicode(text2)))
-                app.todo.clear_text_boxes=1
+                app.todo.clear_text_boxes=app.wordsExist=1
         elif menu_response=="delete" or menu_response=="replace":
             app.set_watch_cursor = 1
             lang2,lang1 = app.toDelete
@@ -1484,19 +1729,19 @@ def gui_event_loop():
             if found and menu_response=="replace": # maybe hack progress.txt as well (taken out of the above loop for better failsafe)
                 d = ProgressDatabase(0)
                 lang2,lang1=lang2.lower(),lang1.lower() # because it's .lower()'d in progress.txt
-                l1find = "!synth:"+lang1.encode('utf-8')+"_"+firstLanguage
+                l1find = S(B("!synth:")+lang1.encode('utf-8')+B("_"+firstLanguage))
                 for item in d.data:
-                    if (item[1]==l1find or (type(item[1])==type([]) and l1find in item[1])) and item[2]=="!synth:"+lang2.encode('utf-8')+"_"+secondLanguage and item[0]:
+                    if (item[1]==l1find or (type(item[1])==type([]) and checkIn(l1find,item[1]))) and item[2]==S(B("!synth:")+lang2.encode('utf-8')+B("_"+secondLanguage)) and item[0]:
                         app.unset_watch_cursor = 1
-                        if not getYN(localise("You have repeated %s=%s %d times.  Do you want to pretend you already repeated %s=%s %d times?") % (lang2,lang1,item[0],t2,t1,item[0])):
+                        if not getYN(localise("You have repeated %s=%s %d times.  Do you want to pretend you already repeated %s=%s %d times?") % (S(lang2),S(lang1),item[0],S(t2),S(t1),item[0])):
                             app.set_watch_cursor = 1 ; break
                         d.data.remove(item)
-                        l1replace = "!synth:"+t2.encode('utf-8')+"_"+firstLanguage
+                        l1replace = S(B("!synth:")+t2.encode('utf-8')+B("_"+firstLanguage))
                         if type(item[1])==type([]):
                             l = item[1]
                             l[l.index(l1find)] = l1replace
                         else: l=l1replace
-                        item = (item[0],l,"!synth:"+t1.encode('utf-8')+"_"+secondLanguage)
+                        item = (item[0],l,S(B("!synth:")+t1.encode('utf-8')+B("_"+secondLanguage)))
                         d.data.append(item)
                         app.set_watch_cursor = 1
                         for i2 in d.unavail:
@@ -1509,69 +1754,85 @@ def gui_event_loop():
             app.todo.clear_text_boxes=1
             app.unset_watch_cursor = 1
             if not found: app.todo.alert = "OOPS: Item to delete/replace was not found in "+vocabFile
-        if app: del app.menu_response
 
 def vocabLinesWithLangs(): # used for merging different users' vocab files
     langs = [secondLanguage,firstLanguage] ; ret = []
-    try: v=u8strip(open(vocabFile,"rb").read()).replace("\r","\n")
-    except IOError: v=""
-    for l in v.split("\n"):
+    try: v=u8strip(read(vocabFile)).replace(B("\r"),B("\n"))
+    except IOError: v=B("")
+    for l in v.split(B("\n")):
         l2=l.lower()
-        if l2.startswith("set language ") or l2.startswith("set languages "): langs=l.split()[2:]
+        if l2.startswith(B("set language ")) or l2.startswith(B("set languages ")): langs=map(S,l.split()[2:])
         elif l: ret.append((tuple(langs),l)) # TODO what about blank lines? (currently they'd be considered duplicates)
     return ret
 
 def appendVocabFileInRightLanguages():
     # check if we need a SET LANGUAGE
     langs = [secondLanguage,firstLanguage]
-    try: v=u8strip(open(vocabFile,"rb").read()).replace("\r","\n")
-    except IOError: v=""
-    for l in v.split("\n"):
+    try: v=u8strip(read(vocabFile)).replace(B("\r"),B("\n"))
+    except IOError: v=B("")
+    for l in v.split(B("\n")):
         l2=l.lower()
-        if l2.startswith("set language ") or l2.startswith("set languages "): langs=l.split()[2:]
-    o=open(vocabFile,"a")
-    if not v.endswith("\n"): o.write("\n")
-    if not langs==[secondLanguage,firstLanguage]: o.write("SET LANGUAGES "+secondLanguage+" "+firstLanguage+"\n")
+        if l2.startswith(B("set language ")) or l2.startswith(B("set languages ")):
+            langs=l.split()[2:]
+            for i in range(len(langs)): langs[i]=S(langs[i])
+    try: o=open(vocabFile,"ab") # (ensure binary on Python 3)
+    except IOError:
+        show_warning("Cannot write to "+vocabFile+" (current directory is "+os.getcwd()+")")
+        return
+    if not v.endswith(B("\n")): o.write(B("\n"))
+    if not langs==[secondLanguage,firstLanguage]: o.write(B("SET LANGUAGES "+secondLanguage+" "+firstLanguage+"\n"))
     return o
 
 def transliterates_differently(text,lang):
     global last_partials_transliteration ; last_partials_transliteration=None
     global partials_are_sporadic ; o=partials_are_sporadic ; partials_are_sporadic = None # don't want to touch the counters here
-    if synthcache_lookup("!synth:"+text+"_"+lang):
+    if synthcache_lookup(B("!synth:")+B(text)+B("_")+B(lang)):
         partials_are_sporadic = o
         if last_partials_transliteration and not last_partials_transliteration==text: return last_partials_transliteration
         else: return # (don't try to translit. if was in synth cache - will have no idea which synth did it)
     partials_are_sporadic = o
     synth=get_synth_if_possible(lang,0) # not to_transliterate=True this time because we want the synth that actually synth'd it (may have done it differently from the transliterating synth)
-    if not synth: return
+    if not synth or not synth.can_transliterate(lang): return
     translit=synth.transliterate(lang,text,forPartials=0)
     if translit and not translit==text: return translit
 
-gui_output_counter = 1
 def gui_outputTo_start():
     if hasattr(app,"outputTo") and app.outputTo.get() and not app.outputTo.get()=="0":
-        global outputFile ; outputFile=None
+        global outputFile,gui_output_directory,oldGID ; outputFile=None
+        if type(gui_output_directory)==type([]):
+            oldGID = gui_output_directory
+            for d in gui_output_directory:
+                if d and d[-1]=="*" and len(os.listdir(d[:-1]))==1: d=d[:-1]+os.listdir(d[:-1])[0]
+                if isDirectory(d):
+                    gui_output_directory = d ; break
+        if type(gui_output_directory)==type([]): gui_output_directory=gui_output_directory[-1]
         try: os.mkdir(gui_output_directory)
         except: pass
-        global gui_output_counter
+        gui_output_counter = 1 # now local because we also got prefix
+        if justSynthesize:
+            if B('#') in B(justSynthesize)[1:]: prefix=B("") # multiple languages
+            else: # prefix the language that's being synth'd
+                prefix=B(justSynthesize).split()[0]
+                if prefix.startswith(B('#')): prefix=prefix[1:]
+        else: prefix = B("lesson")
         while not outputFile or fileExists(outputFile):
-            outputFile=gui_output_directory+os.sep+str(gui_output_counter)+extsep+app.outputTo.get()
+            outputFile=gui_output_directory+os.sep+S(prefix)+str(gui_output_counter)+extsep+app.outputTo.get()
             gui_output_counter += 1
         global write_to_stdout ; write_to_stdout = 0
         global out_type ; out_type = app.outputTo.get()
         global need_run_media_encoder
-        if out_type=="wma" or (out_type=="aac" and not got_program("faac")):
+        if out_type=="wma" or (out_type=="aac" and not (got_program("neroAacEnc") or got_program("faac"))):
             need_run_media_encoder = (out_type,outputFile)
             out_type="wav" ; outputFile=os.tempnam()+dotwav
         else: need_run_media_encoder = 0
-        global soundCollector,viable_synths ; soundCollector,viable_synths = SoundCollector(),[] # may need to re-construct viable_synths when moving from online to offline and vice versa
+        setSoundCollector(SoundCollector())
         global waitBeforeStart, waitBeforeStart_old
         waitBeforeStart_old = waitBeforeStart ; waitBeforeStart = 0
-def gui_outputTo_end():
-    global outputFile, soundCollector, waitBeforeStart, viable_synths
+def gui_outputTo_end(openDir=True):
+    global outputFile, waitBeforeStart, oldGID, gui_output_directory
     if outputFile:
         no_output = not soundCollector.tell() # probably 'no words to put in the lesson'
-        soundCollector = None ; viable_synths = []
+        setSoundCollector(None)
         if no_output: os.remove(outputFile)
         elif need_run_media_encoder:
             t,f = need_run_media_encoder
@@ -1588,9 +1849,10 @@ def gui_outputTo_end():
                 # NB we're passing this to cmd, NOT bash:
                 cmd = "cscript \""+pFiles+"\\Windows Media Components\\Encoder\\WMCmd.vbs\" -input \""+o+"\" -output \""+f+"\" -profile a20_1 -a_content 1"
             elif t=="aac": cmd="afconvert \""+o+"\" -d aac \""+f+"\"" # could also use "afconvert file.wav -d samr file.amr", but amr is bigger than aac and not as good; don't know if anyone has a device that plays amr but not aac.
+            # afconvert default is 64kbit AAC. if want 96+ for music, use -b 96000 after the -d aac (and if want iTunes to be able to accept it, specify extension mp4 instead of aac to afconvert; do not rename aac to mp4, but tell afconvert it's mp4)
             else: assert 0
             if cygwin:
-                assert not "'" in cmd, "apostrophees in pathnames could cause trouble on cygwin"
+                assert not "'" in cmd, "apostrophes in pathnames could cause trouble on cygwin"
                 cmd="echo '"+cmd+" && exit' | cmd" # seems the only way to get it to work on cygwin
             system(cmd)
             os.remove(outputFile)
@@ -1601,47 +1863,52 @@ def gui_outputTo_end():
                 no_output = 1
         outputFile=None
         waitBeforeStart = waitBeforeStart_old
-        if not no_output: openDirectory(gui_output_directory)
+        if openDir and not no_output: openDirectory(gui_output_directory)
+        try: gui_output_directory = oldGID
+        except: pass
 
 def main():
-    global useTK,justSynthesize,waitBeforeStart,traceback,appTitle
+    global useTK,justSynthesize,waitBeforeStart,traceback,appTitle,app,warnings_toprint
     if useTK:
-        if justSynthesize and not justSynthesize[-1]=='*': appTitle=cond('#' in justSynthesize,"Gradint","Reader") # not "language lesson"
+        if justSynthesize and not B(justSynthesize)[-1:]==B('*'): appTitle=cond(B('#') in B(justSynthesize),"Gradint","Reader") # not "language lesson"
         startTk()
-    else: rest_of_main()
+    else:
+        app = None # not False anymore
+        if not appuifw and not android: # REALLY output them to stderr
+            for w in warnings_toprint: show_warning(w)
+        warnings_toprint = [] ; rest_of_main()
 def rest_of_main():
-    global useTK,justSynthesize,waitBeforeStart,traceback,appTitle,saveProgress
-    exitStatus = 0
+    global useTK,justSynthesize,waitBeforeStart,traceback,appTitle,saveProgress,RM_running
+    exitStatus = 0 ; RM_running = 1
 
     try:
         try: ceLowMemory
         except NameError: ceLowMemory=0
         if ceLowMemory and getYN("Low memory! Python may crash. Turn off progress saving for safety?"): saveProgress=0
         
-        if justSynthesize=="-": primitive_synthloop()
-        elif justSynthesize and justSynthesize[-1]=='*':
+        if B(justSynthesize)==B("-"): primitive_synthloop()
+        elif justSynthesize and B(justSynthesize)[-1:]==B('*'):
             justSynthesize=justSynthesize[:-1]
             waitBeforeStart = 0
             just_synthesize() ; lesson_loop()
         elif justSynthesize: just_synthesize()
         elif app and waitBeforeStart: gui_event_loop()
         elif appuifw: s60_main_menu()
+        elif android: android_main_menu()
         else: lesson_loop()
-    except SystemExit: pass
+    except SystemExit:
+        e = sys.exc_info()[1]
+        exitStatus = e.code
     except KeyboardInterrupt: pass
-    except PromptException,prEx:
+    except PromptException:
+        prEx = sys.exc_info()[1]
         waitOnMessage("\nProblem finding prompts:\n"+prEx.message+"\n")
         exitStatus = 1
+    except MessageException:
+        mEx = sys.exc_info()[1]
+        waitOnMessage(mEx.message+"\n") ; exitStatus = 1
     except:
-        w="\nSomething has gone wrong with my program.\nThis is not your fault.\nPlease let me know what it says.\nThanks.  Silas\n"
-        w += str(sys.exc_info()[0])
-        if sys.exc_info()[1]: w += (": "+str(sys.exc_info()[1]))
-        tbObj = sys.exc_info()[2]
-        while tbObj and hasattr(tbObj,"tb_next") and tbObj.tb_next: tbObj=tbObj.tb_next
-        if tbObj and hasattr(tbObj,"tb_lineno"): w += (" at line "+str(tbObj.tb_lineno))
-        if tbObj and hasattr(tbObj,"tb_frame") and hasattr(tbObj.tb_frame,"f_code") and hasattr(tbObj.tb_frame.f_code,"co_filename") and not tbObj.tb_frame.f_code.co_filename.find("gradint"+extsep+"py")>-1: w += (" in "+tbObj.tb_frame.f_code.co_filename+"\n")
-        else: w += (" in "+program_name[:program_name.index("(c)")]+"\n")
-        del tbObj
+        w="\nSomething has gone wrong with my program.\nThis is not your fault.\nPlease let me know what it says.\nThanks.  Silas\n"+exc_info()
         try: import traceback
         except:
             w += "Cannot import traceback\n"
@@ -1657,11 +1924,13 @@ def rest_of_main():
                 if traceback: w += "Details have been written to "+os.getcwd()+os.sep+"last-gradint-error"+extsep+"txt" # do this only if there's a traceback, otherwise little point
             except: pass
         try: # audio warning in case was away from computer.  Do this last as it may overwrite the exception.
+            global soundCollector
+            if app: soundCollector=0
             if not soundCollector and get_synth_if_possible("en",0): synth_event("en","Error in graddint program.").play() # if possible, give some audio indication of the error (double D to try to force correct pronunciation if not eSpeak, e.g. S60)
         except: pass
         waitOnMessage(w.strip())
         if not useTK:
-            if tracebackFile: sys.stderr.write(open("last-gradint-error"+extsep+"txt").read())
+            if tracebackFile: writeB(sys.stderr,read("last-gradint-error"+extsep+"txt"))
             elif traceback: traceback.print_exc() # will be wrong if there was an error in speaking
         exitStatus = 1
         if appuifw: raw_input() # so traceback stays visible
@@ -1674,7 +1943,11 @@ def rest_of_main():
     elif not app==None: pass # (gets here if WAS 'app' but was closed - DON'T output anything to stderr in this case)
     elif appuifw: appuifw.app.set_exit()
     elif riscos_sound: show_info("You may now close this Task Window.\n")
-    else: show_info("\n") # in case got any \r'd string there - don't want to confuse the next prompt
+    elif not android:
+        try:
+            doLabelLastLen ; show_info("\n") # if got any \r'd string there - don't want to confuse the next prompt
+        except NameError: pass # no doLabelLastLen - no \r
+    RM_running = 0
     if exitStatus: sys.exit(exitStatus)
 
 if __name__=="__main__": main() # Note: calling main() is the ONLY control logic that can happen under the 'if __name__=="__main__"' block; everything else should be in main() itself.  This is because gradint-wrapper.exe under Windows calls main() from the exe and does not call this block

gradint-build/src/lessonplan.py → src/lessonplan.py

-# This file is part of the source code of
-# gradint v0.9962 (c) 2002-2010 Silas S. Brown. GPL v3+.
+# This file is part of the source code of Gradint
+# (c) Silas S. Brown.
 #    This program is free software; you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by
 #    the Free Software Foundation; either version 3 of the License, or
@@ -25,29 +25,34 @@ class ProgressDatabase(object):
           self.unavail = mergeProgress(self.data,scanSamples()+parseSynthVocab(vocabFile))
           if not cache_maintenance_mode:
             doLabel("Checking transliterations")
+            global tList # for Python 2.1
             tList = {}
-            def addVs(ff,dirBase):
-                if dirBase: dirBase += os.sep
-                if dirBase+ff in variantFiles:
-                   if os.sep in ff: ffpath=ff[:ff.rfind(os.sep)+1]
-                   else: ffpath=""
-                   variantList=map(lambda x:ffpath+x,variantFiles[dirBase+ff])
+            def addVs(ff,dirBase): # add variants of ff to tList which we might need to transliterate
+                dirBase,ff = B(dirBase),B(ff)
+                if dirBase: dirBase += B(os.sep)
+                if checkIn(dirBase+ff,variantFiles):
+                   if B(os.sep) in ff: ffpath=ff[:ff.rfind(B(os.sep))+1]
+                   else: ffpath=B("")
+                   variantList=map(lambda x,f=ffpath:f+B(x),variantFiles[dirBase+ff])
                 else: variantList = [ff]
                 l=languageof(ff)
                 for f in variantList:
-                  if f.lower().endswith(dottxt): text=u8strip(open(dirBase+f,"rb").read()).strip(wsp)
-                  elif f.find("!synth")==-1: continue # don't need to translit. filenames of wav's etc
+                  f = B(f)
+                  if f.lower().endswith(B(dottxt)):
+                      text=bwspstrip(u8strip(read(dirBase+f)))
+                  elif f.find(B("!synth"))==-1: continue # don't need to translit. filenames of wav's etc
                   else: text = textof(f)
-                  if not l in tList: tList[l]={}
+                  if not checkIn(l,tList): tList[l]={}
                   tList[l][text]=1
             for ff in availablePrompts.lsDic.values(): addVs(ff,promptsDirectory)
             for _,l1,l2 in self.data:
                 if not type(l1)==type([]): l1=[l1]
                 for ff in l1+[l2]: addVs(ff,samplesDirectory)
             doLabel("Transliterating")
-            for lang,dic in tList.items():
+            for lang,dic in list(tList.items()):
                 s = get_synth_if_possible(lang,0)
-                if s and hasattr(s,"update_translit_cache"): s.update_translit_cache(lang,dic.keys())
+                if s and hasattr(s,"update_translit_cache"): s.update_translit_cache(lang,list(dic.keys()))
+            del tList
         self.didScan = alsoScan
     def _load_from_binary(self):
         if pickledProgressFile and fileExists(pickledProgressFile):
@@ -55,11 +60,12 @@ class ProgressDatabase(object):
                 global firstLanguage, secondLanguage, otherLanguages
                 if compress_progress_file or (unix and got_program("gzip")):
                     if paranoid_file_management: open(pickledProgressFile) # ensure ready
-                    f = os.popen('gzip -fdc "'+pickledProgressFile+'"',"rb")
+                    f = os.popen('gzip -fdc "'+pickledProgressFile+'"',popenRB)
                 else: f=open(pickledProgressFile,"rb")
                 try: thingsToSet, tup = pickle.Unpickler(f).load()
                 except: return False # probably moved to a different Python version or something
                 exec(thingsToSet)
+                self._py3_fix()
                 return True
             # otherwise drop out and return None
     def _load_from_text(self,fromString=0):
@@ -67,11 +73,11 @@ class ProgressDatabase(object):
         elif fileExists(progressFile):
             if compress_progress_file or (unix and got_program("gzip")):
                 if paranoid_file_management: open(progressFile) # ensure ready
-                expr = os.popen('gzip -fdc "'+progressFile+'"',"rb").read()
-            else: expr = open(progressFile,"rb").read()
+                expr = readB(os.popen('gzip -fdc "'+progressFile+'"',popenRB))
+            else: expr = read(progressFile)
         else: expr = None
         if expr:
-            expr = u8strip(expr) # just in case progress.txt has been edited in Notepad
+            expr = u8strip(expr).replace(B("\r\n"),B("\n")) # just in case progress.txt has been edited in Notepad
             # First, try evaluating it as self.data (legacy progress.txt from older versions).  If that doesn't work, execute it (newer versions).
             global firstLanguage, secondLanguage, otherLanguages
             try: self.data = eval(expr)
@@ -81,64 +87,102 @@ class ProgressDatabase(object):
                 except: codeop = 0
                 if codeop: # try a lower-memory version (in case text file has been edited by hand and we're on NSLU2 or something) - don't compile all of it at once
                     lineCache = []
-                    for l in expr.replace("\r\n","\n").split("\n"):
+                    for l in expr.replace(B("\r\n"),B("\n")).split(B("\n")):
                         lineCache.append(l)
-                        if lineCache[-1].endswith(","): continue # no point trying to compile if it's obviously incomplete
-                        code = codeop.compile_command("\n".join(lineCache))
+                        if lineCache[-1].endswith(B(",")): continue # no point trying to compile if it's obviously incomplete
+                        code = codeop.compile_command("# coding=utf-8\n"+S(B("\n").join(lineCache)))
                         if code:
                             lineCache = []
-                            exec code
-                else: exec(expr)
+                            exec(code)
+                else: exec(B("# coding=utf-8\n")+expr)
             del expr
         # Remove legacy extentions in promptsData (needed only when loading from text, as this was before pickledProgressFile was added)
-        for k in self.promptsData.keys():
+        for k in list(self.promptsData.keys()):
             if k.endswith(dotwav) or k.endswith(dotmp3):
                 self.promptsData[k[:-len(dotwav)]]=self.promptsData[k]
                 del self.promptsData[k]
+        self._py3_fix()
+    def _saved_by_py3(self):
+        # NB the Windows version of Gradint is still Python 2.3 so generator expressions (new in 2.4) would be a syntax error even though this code is never reached in that version, so:
+        for l in [self.data,self.unavail]:
+            for i in l:
+                for j in i[1:]:
+                    if type(j)==str: j=[j]
+                    for k in j:
+                        for c in k:
+                            if ord(c) > 255: return True # must have been written by the Python 3 version
+    def _py3_fix(self):
+        if not type("")==type(u""): return
+        # we're Python 3, and we might have just loaded data from Python 2.  Might have to encode as Latin-1 then decode as UTF-8.  But don't do this if file was in fact saved by Python 3.
+        if self._saved_by_py3(): return
+        for l in [self.data,self.unavail]:
+            for i in range(len(l)):
+                for j in [1,2]:
+                    if type(l[i][j])==str: l[i]=l[i][:j]+(S2(LB(l[i][j])),)+l[i][j+1:]
+                    elif type(l[i][j])==list: l[i]=l[i][:j]+(map(lambda x:S2(LB(x)),l[i][j]),)+l[i][j+1:]
+    def _py3_fix_on_save(self):
+        if type("")==type(u"") and not self._saved_by_py3(): self.unavail.append((1,u"\u2014","[Py3]")) # ensure there's at least one, to prevent a py3_fix redo
     def save(self,partial=0):
         if need_say_where_put_progress: show_info("Saving "+cond(partial,"partial ","")+"progress to "+progressFile+"... ")
         else: show_info("Saving "+cond(partial,"partial ","")+"progress... ")
+        self._py3_fix_on_save()
         global progressFileBackup
         # Remove 0-repeated items (helps editing by hand)
         data = [] # don't use self.data - may want to make another lesson after saving
         for a,b,c in self.data:
             if a: data.append(denumber_filelists(a,b,c))
-        data.sort(cmpfunc) # to normalise when using diff etc
+        sort(data,cmpfunc) # to normalise when using diff etc
         if progressFileBackup:
             try:
                 import shutil
                 shutil.copy2(progressFile,progressFileBackup) # preserve timestamp etc if shutil is available
             except:
-                try:
-                    open(progressFileBackup,'wb').write(open(progressFile,'rb').read())
+                try: write(progressFileBackup,read(progressFile))
                 except IOError: pass # maybe progressFile not made yet
             progressFileBackup = None
         while True:
           try:
-            if compress_progress_file: f=os.popen('gzip -9 > "'+progressFile+'"','w')
+            if compress_progress_file:
+              if paranoid_file_management: fn=os.tempnam() # on some ftpfs setups gzip can fail causing silent corruption
+              else: fn=progressFile
+              f=os.popen('gzip -9 > "'+fn+'"','w')
             else: f = open(progressFile,'w')
+            global progressFileHeader
+            if type(u"")==type(""): # Python 3: ensure UTF-8
+                import codecs
+                realF,f = f,codecs.getwriter("utf-8")(f.buffer)
+                progressFileHeader=progressFileHeader.replace("mode: python ","mode: python; coding: utf-8")
+            else: realF = f
             f.write(progressFileHeader)
             f.write("firstLanguage=\"%s\"\nsecondLanguage=\"%s\"\n# otherLanguages=%s\n" % (firstLanguage,secondLanguage,otherLanguages)) # Note: they're declared "global" above (and otherLanguages commented out here for now, since may add to it in advanced.txt) (Note also save_binary below.)
-            if self.didScan: f.write("# collection=%d done=%d left=%d lessonsLeft=%d\n" % (len(self.data),len(data),len(self.data)-len(data),(len(self.data)-len(data)+maxNewWords-1)/maxNewWords))
+            if self.didScan and maxNewWords: f.write("# collection=%d done=%d left=%d lessonsLeft=%d\n" % (len(self.data),len(data),len(self.data)-len(data),(len(self.data)-len(data)+maxNewWords-1)/maxNewWords))
             prettyPrintLongList(f,"self.data",data)
             f.write("self.promptsData=") ; pprint.PrettyPrinter(indent=2,width=60,stream=f).pprint(self.promptsData)
             prettyPrintLongList(f,"self.unavail",self.unavail)
-            f.close()
+            realF.close()
+            if compress_progress_file and paranoid_file_management: write(progressFile,read(fn)),os.remove(fn)
             self.save_binary(data)
           except IOError: # This can happen for example on some PocketPC devices if you reconnect the power during progress save (which is likely if you return the device to the charger when lesson finished)
-            if app or appuifw:
+            if app or appuifw or android:
               if getYN("I/O fault when saving progress. Retry?"): continue
               # TODO else try to restore the backup?
             else: raise
           break
         if not partial: self.saved_completely = 1
-        if not app and not appuifw: show_info("done\n")
+        if not app and not appuifw and not android: show_info("done\n")
     def save_binary(self,data): # save a pickled version if possible (no error if not)
-        if not pickledProgressFile or not pickle: return
+        if not (pickledProgressFile and pickle): return
+        self._py3_fix_on_save()
         try:
-            if compress_progress_file: f=os.popen('gzip -9 > "'+pickledProgressFile+'"','wb')
+            if compress_progress_file:
+              if paranoid_file_management: fn=os.tempnam()
+              else: fn=pickledProgressFile # TODO near-duplicate code with above
+              f=os.popen('gzip -9 > "'+fn+'"',popenWB)
+              if hasattr(f,'buffer'): _,f = f,f.buffer
             else: f = open(pickledProgressFile,'wb')
             pickle.Pickler(f,-1).dump(("self.data,self.promptsData,self.unavail,firstLanguage,secondLanguage = tup", (data,self.promptsData,self.unavail,firstLanguage,secondLanguage)))
+            f.close()
+            if compress_progress_file and paranoid_file_management: write(pickledProgressFile,read(fn)),os.remove(fn)
         except IOError: pass # OK if not got permissions to do it (NB need to catch the write as well because popen won't throw, and don't have to worry about a corrupted partial binary because loader would ignore it)
     def savePartial(self,filesNotPlayed):
         curPD,curDat = self.promptsData, self.data[:] # in case want to save a more complete one later
@@ -146,7 +190,7 @@ class ProgressDatabase(object):
         if hasattr(self,"previous_filesNotPlayed"):
             i=0
             while i<len(filesNotPlayed):
-                if filesNotPlayed[i] in self.previous_filesNotPlayed: i+=1
+                if checkIn(filesNotPlayed[i],self.previous_filesNotPlayed): i+=1
                 else: del filesNotPlayed[i] # cumulative effects if managed to play it last time but not this time (and both lessons incomplete)
         self.previous_filesNotPlayed = filesNotPlayed = list2set(filesNotPlayed)
         if not filesNotPlayed:
@@ -160,17 +204,17 @@ class ProgressDatabase(object):
             l.append(self.data[i][2])
             found=0
             for ii in l:
-              if ii in filesNotPlayed:
+              if checkIn(ii,filesNotPlayed):
                   self.data[i] = self.oldData[i]
                   found=1 ; break
             if not found and not self.data[i] == self.oldData[i]: changed = 1
         if changed: self.save(partial=1)
-        elif app==None and not appuifw: show_info("No sequences were fully complete so no changes saved\n")
+        elif app==None and not appuifw and not android: show_info("No sequences were fully complete so no changes saved\n")
         self.promptsData,self.data = curPD,curDat
     def makeLesson(self):
         global maxLenOfLesson
         self.l = Lesson()
-        self.data.sort(cmpfunc) ; jitter(self.data)
+        sort(self.data,cmpfunc) ; jitter(self.data)
         self.oldData = self.data[:] # for handling interrupts & partial progress saves
         self.exclude = {} ; self.do_as_poem = {}
         # First priority: Recently-learned old words
@@ -192,28 +236,34 @@ class ProgressDatabase(object):
         for p in poems:
             for l in p: self.do_as_poem[self.responseIndex[l]] = p
         self.addToLesson(reallyKnownThreshold,-1,1,1,-1)
+        if not self.l.events:
+            # desperado if someone really wants extra revision of few words
+            global randomDropLevel, randomDropLevel2
+            rdl,rdl2,randomDropLevel,randomDropLevel2 = randomDropLevel,randomDropLevel2,0,0
+            self.addToLesson(reallyKnownThreshold,-1,1,1,-1)
+            randomDropLevel, randomDropLevel2 = rdl,rdl2
         l = self.l ; del self.l, self.responseIndex, self.do_as_poem
         if not l.events: raise Exception("Didn't manage to put anything in the lesson")
-        if commentsToAdd: l.addSequence(commentSequence())
+        if commentsToAdd: l.addSequence(commentSequence(),False)
         if orderlessCommentsToAdd:
             for c in orderlessCommentsToAdd:
                 try:
-                    l.addSequence([GluedEvent(Glue(1,maxLenOfLesson),fileToEvent(c,""))])
+                    l.addSequence([GluedEvent(Glue(1,maxLenOfLesson),fileToEvent(c,""))],False)
                 except StretchedTooFar:
-                    sys.stderr.write("Was trying to add %s\n" % (c,))
+                    show_info(("Was trying to add %s\n" % (c,)),True)
                     raise
         # Add note on "long pause", for beginners
         longpause = "longpause_"+firstLanguage
-        if not advancedPromptThreshold and not longpause in availablePrompts.lsDic: longpause = "longpause_"+secondLanguage
+        if not advancedPromptThreshold and not checkIn(longpause,availablePrompts.lsDic): longpause = "longpause_"+secondLanguage
         o=maxLenOfLesson ; maxLenOfLesson = max(l.events)[0]
-        if longpause in availablePrompts.lsDic and self.promptsData.get(longpause,0)==0:
+        if checkIn(longpause,availablePrompts.lsDic) and self.promptsData.get(longpause,0)==0:
             try:
-                def PauseEvent(): return fileToEvent(availablePrompts.lsDic[longpause],promptsDirectory)
-                firstPauseMsg = PauseEvent()
+                def PauseEvent(longpause): return fileToEvent(availablePrompts.lsDic[longpause],promptsDirectory)
+                firstPauseMsg = PauseEvent(longpause)
                 # the 1st potentially-awkward pause is likely to be a beepThreshold-length one
-                l.addSequence([GluedEvent(Glue(1,maxLenOfLesson),CompositeEvent([firstPauseMsg,Event(max(5,beepThreshold-firstPauseMsg.length))]))])
+                l.addSequence([GluedEvent(Glue(1,maxLenOfLesson),CompositeEvent([firstPauseMsg,Event(max(5,beepThreshold-firstPauseMsg.length))]))],False)
                 while True:
-                    l.addSequence([GluedEvent(Glue(1,maxLenOfLesson),CompositeEvent([PauseEvent(),Event(50)]))])
+                    l.addSequence([GluedEvent(Glue(1,maxLenOfLesson),CompositeEvent([PauseEvent(longpause),Event(50)]))],False)
                     self.promptsData[longpause] = 1
             except StretchedTooFar: pass
         maxLenOfLesson = o
@@ -246,7 +296,7 @@ class ProgressDatabase(object):
             numFailures = 0 ; startTime = time.time() # for not taking too long
             for i in xrange(len(self.data)):
                 if maxNumToAdd>-1 and numberAdded >= maxNumToAdd: break # too many
-                if i in self.exclude: continue # already had it
+                if checkIn(i,self.exclude): continue # already had it
                 (timesDone,promptFile,zhFile)=self.data[i]
                 if timesDone < minTimesDone or (maxTimesDone>=0 and timesDone > maxTimesDone): continue # out of range this time
                 if timesDone >= knownThreshold: thisNumToTry = min(random.choice([2,3,4]),numToTry)
@@ -255,17 +305,17 @@ class ProgressDatabase(object):
                     # dropping it at random
                     self.exclude[i] = 1 # pretend we've done it
                     continue
-                if i in self.do_as_poem:
+                if checkIn(i,self.do_as_poem):
                     # this is part of a "known poem" and let's try to do it in sequence
                     self.try_add_poem(self.do_as_poem[i]) ; continue
                 oldPromptsData = self.promptsData.copy()
                 seq=anticipationSequence(promptFile,zhFile,timesDone,timesDone+thisNumToTry,self.promptsData,introductions(zhFile,self.data))
-                seq[0].timesDone = timesDone # for diagram.py
+                seq[0].timesDone = timesDone # for diagram.py (and now status messages) to know if it's a new word
                 global earliestAllowedEvent ; earliestAllowedEvent = 0
                 if not timesDone and type(promptFile)==type([]):
                     # for poems: if any previously-added new word makes part of the prompt, try to ensure this one is introduced AFTER that one
-                    for f,t in newWordTimes.items():
-                        if f in promptFile: earliestAllowedEvent = max(earliestAllowedEvent,t)
+                    for f,t in list(newWordTimes.items()):
+                        if checkIn(f,promptFile): earliestAllowedEvent = max(earliestAllowedEvent,t)
                 if not timesDone: newWordTimes[zhFile] = maxLenOfLesson # by default (couldn't fit it in).  (add even if not type(promptFile)==type([]), because it might be a first line)
                 try: self.l.addSequence(seq)
                 except StretchedTooFar: # If this happens, couldn't fit the word in anywhere.  If this is "filling in gaps" then it's likely that we won't be able to fit in any more words this lesson, so stop trying.
@@ -285,8 +335,8 @@ class ProgressDatabase(object):
                 numberAdded = numberAdded + 1
                 self.exclude[i] = 1
                 # Keep a count
-                if not timesDone: self.l.newWords=self.l.newWords + 1
-                else: self.l.oldWords=self.l.oldWords+1
+                if not timesDone: self.l.newWords += 1
+                else: self.l.oldWords += 1
                 self.data[i]=(timesDone+thisNumToTry,promptFile,zhFile)
                 if not timesDone: newWordTimes[zhFile] = seq[0].getEventStart(0) # track where it started
         return numberAdded
@@ -301,8 +351,11 @@ class ProgressDatabase(object):
             poemSequence.append(Event(e.length))
             poemSequence.append(e)
             self.exclude[self.responseIndex[line]] = 1 # (don't try to add it again this lesson, whether successful or not)
-        try: self.l.addSequence([GluedEvent(initialGlue(),CompositeEvent(poemSequence))])
+        poemSequence = [GluedEvent(initialGlue(),CompositeEvent(poemSequence))]
+        poemSequence[0].endseq = False # boolean 'is it a new word'
+        try: self.l.addSequence(poemSequence)
         except StretchedTooFar: return
+        self.l.oldWords += 1 # have to only count it as one due to endseq handling
         for line in poem: self.data[self.responseIndex[line]]=(self.data[self.responseIndex[line]][0]+1,)+self.data[self.responseIndex[line]][1:]
     def veryExperienced(self):
         # used for greater abbreviation in the prompts etc
@@ -311,7 +364,7 @@ class ProgressDatabase(object):
             covered = 0
             for timesDone,promptFile,zhFile in self.data:
                 if timesDone: covered += 1
-            x = (covered > 1000) # arbitrary
+            x = (covered > veryExperiencedThreshold)
             self.cached_very_experienced = x
         return x
     def message(self):
@@ -320,8 +373,8 @@ class ProgressDatabase(object):
         for timesDone,promptFile,zhFile in self.data:
             if timesDone:
                 covered += 1
-                if zhFile.find(exclude_from_coverage)==-1: actualCovered += 1
-            if zhFile.find(exclude_from_coverage)==-1: actualTotal += 1
+                if B(zhFile).find(B(exclude_from_coverage))==-1: actualCovered += 1
+            if B(zhFile).find(B(exclude_from_coverage))==-1: actualTotal += 1
         l=cond(app,localise,lambda x:x)
         toRet = (l("You have %d words in your collection") % total)
         if not total==actualTotal: toRet += (" (actually %d)" % actualTotal)
@@ -336,15 +389,23 @@ def prettyPrintLongList(f,thing,data):
     if winCEsound: p=0 # don't use WinCE's PrettyPrinter here - it inconsistently escapes utf8 sequences (result can't reliably be edited in MADE etc)
     else: p=pprint.PrettyPrinter(indent=2,width=60,stream=f)
     for start in range(0,len(data),step):
+        dat = data[start:start+step]
+        if type("")==type(u""): # Python 3: probably best to output strings rather than bytes
+            for i in range(len(dat)):
+                for j in [1,2]:
+                    if type(dat[i][j])==bytes:
+                        dat[i]=dat[i][:j]+(S2(dat[i][j]),)+dat[i][j+1:]
+                    elif type(dat[i][j])==list:
+                        dat[i]=dat[i][:j]+(map(S2,dat[i][j]),)+dat[i][j+1:]
         if start: f.write(thing+"+=")
         else: f.write(thing+"=")
         if p:
             t = time.time()
-            p.pprint(data[start:start+step])
+            p.pprint(dat)
             if not start and (time.time()-t)*(len(data)/step) > 5: p=0 # machine is too slow - use faster version on next iteration
         else: # faster version - not quite as pretty
             f.write("[")
-            for d in data[start:start+step]: f.write("  "+repr(d)+",\n")
+            for d in dat: f.write("  "+repr(d)+",\n")
             f.write("]\n")
 
 def calcDropLevel(timesDone):
@@ -369,15 +430,23 @@ def cmpfunc(x,y):
     if r: return r # skipping the rest if x[0]!=y[0]
     if x[0]: return cmpfunc_test(x,y) # our special order is needed only for new words (to ensure correct order of introduction)
     def my_toString(x):
-        if type(x)==type([]): return "".join(x)
-        else: return x
-    x2 = (my_toString(x[1]).replace(os.sep,chr(0)), my_toString(x[2]).replace(os.sep,chr(0)))
-    y2 = (my_toString(y[1]).replace(os.sep,chr(0)), my_toString(y[2]).replace(os.sep,chr(0)))
+        if type(x)==type([]): return B("").join(map(B,x))
+        else: return B(x)
+    x2 = (my_toString(x[1]).replace(B(os.sep),chr(0)), my_toString(x[2]).replace(B(os.sep),chr(0)))
+    y2 = (my_toString(y[1]).replace(B(os.sep),chr(0)), my_toString(y[2]).replace(B(os.sep),chr(0)))
     return cmpfunc_test(x2,y2)
 def cmpfunc_test(x,y):
-    if x < y: return -1
-    elif x > y: return 1
-    else: return 0
+    try:
+        if x < y: return -1
+        elif x > y: return 1
+        else: return 0
+    except: # probably Python 3 can't compare list vs string
+        if x[0] < y[0]: return -1
+        elif x[0] > y[0]: return 1
+        x,y = repr(x),repr(y)
+        if x < y: return -1
+        elif x > y: return 1
+        else: return 0
 
 def denumber_filelists(r,x,y):
     if type(x)==type([]): x=map(lambda z:denumber_synth(z),x)
@@ -386,24 +455,24 @@ def denumber_filelists(r,x,y):
     else: y=denumber_synth(y)
     return (r,x,y)
 def denumber_synth(z,also_norm_extsep=0):
-    zf = z.find("!synth:")
-    if zf>-1:
-        z=z[zf:].lower() # so ignores the priority-number it had (because the vocab.txt file might have been re-organised hence changing all the numbers).  Also a .lower() so case changes don't change progress.  (Old versions of gradint said .lower() when parsing vocab.txt, but this can cause problems with things like Mc[A-Z].. in English espeak)
-        if z.endswith(dotwav) or z.endswith(dotmp3): return z[:z.rindex(extsep)] # remove legacy extensions from synth vocab
-    elif also_norm_extsep: return z.replace("\\","/").replace(".","/") # so compares equally across platforms with os.sep and extsep differences
+    z=B(z) ; zf = z.find(B("!synth:"))
+    if zf>=0:
+        z=lower(z[zf:]) # so ignores the priority-number it had (because the vocab.txt file might have been re-organised hence changing all the numbers).  Also a .lower() so case changes don't change progress.  (Old versions of gradint said .lower() when parsing vocab.txt, but this can cause problems with things like Mc[A-Z].. in English espeak)
+        if z.endswith(B(dotwav)) or z.endswith(B(dotmp3)): return z[:z.rindex(B(extsep))] # remove legacy extensions from synth vocab
+    elif also_norm_extsep: return z.replace(B("\\"),B("/")).replace(B("."),B("/")) # so compares equally across platforms with os.sep and extsep differences
     return z
 
+def norm_filelist(x,y):
+    def noext(x): return (B(x)+B(' '))[:B(x).rfind(B(extsep))] # so user can change e.g. wav to mp3 without disrupting progress.txt (the ' ' is simply removed if rfind returns -1)
+    if type(x)==type([]): x=tuple(map(lambda z,noext=noext:denumber_synth(noext(z),1),x))
+    else: x=denumber_synth(noext(x),1)
+    if type(y)==type([]): y=tuple(map(lambda z,noext=noext:denumber_synth(noext(z),1),y))
+    else: y=denumber_synth(noext(y),1)
+    return (x,y)
 def mergeProgress(progList,scan):
     # Merges a progress database with a samples scan, to
     # pick up any new samples that were added since last
     # time.  Appends to progList.  Return value see below.
-    def norm_filelist(x,y):
-        def noext(x): return (x+extsep)[:x.rfind(extsep)] # so user can change e.g. wav to mp3 without disrupting progress.txt
-        if type(x)==type([]): x=tuple(map(lambda z:denumber_synth(noext(z),1),x))
-        else: x=denumber_synth(noext(x),1)
-        if type(y)==type([]): y=tuple(map(lambda z:denumber_synth(noext(z),1),y))
-        else: y=denumber_synth(noext(y),1)
-        return (x,y)
     proglistDict = {} ; scanlistDict = {} ; n = 0
     while n<len(progList):
         i,j,k = progList[n]
@@ -423,7 +492,7 @@ def mergeProgress(progList,scan):
     renames = {}
     for (_,j,k) in scan:
         key = norm_filelist(j,k)
-        if key in proglistDict:
+        if checkIn(key,proglistDict):
             # an existing item - but in the case of synth'd vocab, we need to take the capitals/lower-case status from the scan rather than from the progress file (see comment above in denumber_synth) so:
             progList[proglistDict[key]]=(progList[proglistDict[key]][0],j,k)
         elif type(key[0])==type("") and (key[0]+key[1]).find("!synth")==-1 and ("_" in key[0] and "_" in key[1]):
@@ -436,16 +505,16 @@ def mergeProgress(progList,scan):
                 while ki>lastDirsep and not "0"<=normK[ki]<="9": ki -= 1
                 if ki<=lastDirsep: break
                 key2 = (key[0][:ki+1]+key[0][key[0].rindex("_"):],key[1][:ki+1]+key[1][key[1].rindex("_"):])
-                if key2 in proglistDict:
-                    if not key2 in renames: renames[key2] = []
+                if checkIn(key2,proglistDict):
+                    if not checkIn(key2,renames): renames[key2] = []
                     renames[key2].append((j,k))
                     found=1 ; break
                 while ki>lastDirsep and "0"<=normK[ki]<="9": ki -= 1
             if not found: progList.append((0,j,k)) # new item
         else: progList.append((0,j,k)) # ditto
         scanlistDict[key]=1
-    for k,v in renames.items():
-        if k in scanlistDict or len(v)>1: # can't make sense of this one - just add the new stuff
+    for k,v in list(renames.items()):
+        if checkIn(k,scanlistDict) or len(v)>1: # can't make sense of this one - just add the new stuff
             for jj,kk in v: progList.append((0,jj,kk))
         else: progList[proglistDict[k]]=(progList[proglistDict[k]][0],v[0][0],v[0][1])
     # finally, separate off any with non-0 progress that are
@@ -455,7 +524,7 @@ def mergeProgress(progList,scan):
     n = 0 ; unavailList = []
     while n<len(progList):
         i,j,k = progList[n]
-        if not norm_filelist(j,k) in scanlistDict:
+        if not checkIn(norm_filelist(j,k), scanlistDict):
             unavailList.append((i,j,k))
             del progList[n]
         else: n += 1
@@ -511,11 +580,11 @@ def jitter(list):
     # need to work on them more quickly, and can limit manually
     limitCounts = {} ; i = 0 ; imax = len(list)
     while i < imax:
-        if list[i][0]==0 and (list[i][-1] in limitedFiles): # or not languageof(list[i][2])==secondLanguage):
+        if list[i][0]==0 and checkIn(list[i][-1],limitedFiles): # or not languageof(list[i][2])==secondLanguage):
             # if not languageof(list[i][2])==secondLanguage: countNo="other-langs"
             # else:
             countNo = limitedFiles[list[i][-1]]
-            if not countNo in limitCounts: limitCounts [countNo] = 0
+            if not checkIn(countNo,limitCounts): limitCounts [countNo] = 0
             limitCounts [countNo] += 1
             # (below is a hack: if already moved something, set limit_words to 1.  May want to do it better than that e.g. go back and ensure the first thing only left 1 as well, or share out limit_words among any concurrently available new items that are just about to be introduced)
             if limitCounts [countNo] > cond(imax==len(list),limit_words,1) or (countNo=="other-langs" and limitCounts [countNo] > 1):
@@ -540,14 +609,14 @@ def find_known_poems(progressData):
             nextLineDic[line]=response # TODO check what would happen if 2 different poems in vocab.txt share an identical line (or if responseIndex is ambiguous in any way)
             hasPreviousLine[response]=True
     poems = []
-    for poemFirstLine in filter(lambda x:not x in hasPreviousLine,nextLineDic.keys()):
+    for poemFirstLine in filter(lambda x,hasPreviousLine=hasPreviousLine:not x in hasPreviousLine,nextLineDic.keys()):
         poemLines = [] ; line = poemFirstLine
         poem_is_viable = True
         while True:
             poemLines.append(line)
-            if not line in responseIndex or progressData[responseIndex[line]][0] < reallyKnownThreshold:
+            if not checkIn(line,responseIndex) or progressData[responseIndex[line]][0] < reallyKnownThreshold:
                 poem_is_viable = False ; break # whole poem not in database yet, or is but not well-rehearsed
-            if not line in nextLineDic: break
+            if not checkIn(line,nextLineDic): break
             line = nextLineDic[line]
         if poem_is_viable: poems.append(poemLines)
     return poems, responseIndex

gradint-build/src/loop.py → src/loop.py

-# This file is part of the source code of
-# gradint v0.9962 (c) 2002-2010 Silas S. Brown. GPL v3+.
+# This file is part of the source code of Gradint
+# (c) Silas S. Brown.
 #    This program is free software; you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by
 #    the Free Software Foundation; either version 3 of the License, or
@@ -18,7 +18,12 @@ def doOneLesson(dbase):
         lesson = dbase.makeLesson()
     else:
         soFar = "Re-loading saved lesson, so not scanning collection."
-        if compress_progress_file: lesson=pickle.Unpickler(os.popen('gzip -fdc "'+saveLesson+'"','rb')).load()
+        if compress_progress_file:
+            pp = os.popen('gzip -fdc "'+saveLesson+'"',popenRB)
+            if hasattr(pp,'buffer'): ppb = pp.buffer
+            else: ppb = pp
+            lesson=pickle.Unpickler(ppb).load()
+            del ppb,pp
         else: lesson=pickle.Unpickler(open(saveLesson,'rb')).load()
     if app and not dbase: app.setNotFirstTime()
     while 1:
@@ -31,24 +36,44 @@ def doOneLesson(dbase):
             waitOnMessage(msg+interrupt_instructions())
             #time.sleep(5)
             time.sleep(2) # less confusing for beginners
-        elif not app and not appuifw: show_info(msg+interrupt_instructions()+"\n",True)
+        elif not app and not appuifw and not android: show_info(msg+interrupt_instructions()+"\n",True)
         if startFunction: startFunction()
         if app:
             app.setLabel("Starting lesson")
             app.cancelling = 0
         lesson.play()
+      if not gluedListTracker==None:
+          global lastLessonMade ; lastLessonMade = lesson # used by utils/trace.py
       if dbase and saveProgress and not dbase.saved_completely: # justSaveLesson is a no-op if not first time through lesson (because scripts that use it probably mean "save if not already save"; certainly don't mean "play if is saved")
           if cancelledFiles: dbase.savePartial(cancelledFiles)
           else: dbase.save()
           if dbase.saved_completely and app: app.setNotFirstTime() # dbase.saved_completely could have been done by EITHER of the above (e.g. overlapping partial saves)
           if saveLesson:
-              if compress_progress_file: pickle.Pickler(os.popen('gzip -9 > "'+saveLesson+'"','wb'),-1).dump(lesson)
+              if compress_progress_file:
+                  pp = os.popen('gzip -9 > "'+saveLesson+'"',popenWB)
+                  if hasattr(pp,'buffer'): ppb=pp.buffer
+                  else: ppb = pp
+                  pickle.Pickler(ppb,-1).dump(lesson) # TODO: paranoid_file_management ? (c.f. saveProgress)
+                  del ppb,pp
               else: pickle.Pickler(open(saveLesson,"wb"),-1).dump(lesson)
               saveLesson = None # so saves only the first when doing multiple lessons
               if justSaveLesson: break
       if not app and not app==None: break # close box pressed
       if not waitBeforeStart or not getYN(cond(not askAgain_explain and (not dbase or not saveProgress or dbase.saved_completely),"Hear this lesson again?",askAgain_explain+"Start this lesson again?")): break
 
+def disable_lid(restore): # for portable netbooks (like eee), can close lid & keep listening
+  if unix:
+   if app and not outputFile:
+    global oldLid,warnedAC
+    try: warnedAC
+    except: warnedAC=0
+    if (not restore) and getoutput("cat /proc/acpi/ac_adapter/AC*/state 2>/dev/null").find("off-line")>=0 and not warnedAC:
+      waitOnMessage("Some quirky Linux battery managers turn speakers off mid-lesson, so AC power is recommended.") ; warnedAC=1 # (TODO what if pull out AC during the lesson without looking at the screen?  Spoken message??)
+    ls = "et org.gnome.settings-daemon.plugins.power lid-close-" ; src=["ac","battery"]
+    if restore and oldLid[0]: return [getoutput("gsettings s"+ls+p+"-action "+q+" 2>/dev/null") for p,q in zip(src,oldLid)]
+    oldLid = [getoutput("gsettings g"+ls+p+"-action 2>/dev/null").replace("'","") for p in src]
+    if oldLid[0]: [getoutput("gsettings s"+ls+p+"-action blank 2>/dev/null") for p in src]
+
 if loadLesson==-1: loadLesson=(fileExists(saveLesson) and time.localtime(os.stat(saveLesson).st_mtime)[:3]==time.localtime()[:3])
 
 def lesson_loop():
@@ -56,6 +81,7 @@ def lesson_loop():
   if ask_teacherMode and not soundCollector and waitBeforeStart: teacherMode=getYN("Use teacher assistant mode? (say 'no' for self-study)")
   try:
     # doLabel("Scanning prompts") # rarely takes long even on low-end systems
+    init_scanSamples() # in case was messed around with before
     availablePrompts = AvailablePrompts() # here so app is already initialised before any warnings
     global dbase # so can be accessed by interrupt handler
     if loadLesson: dbase=None
@@ -64,13 +90,11 @@ def lesson_loop():
         dbase = ProgressDatabase()
         if not dbase.data:
             msg = "There are no words to put in the lesson."
-            if app or appuifw:
+            if app or appuifw or android:
                 drop_to_synthloop = False
-                msg = localise(msg)
-                if app and hasattr(app,"TestButton"): msg += ("\n"+(localise("Please press \"%s\" first.") % localise("Manage word list")))
-                else: msg += ("\n"+localise("Please add some words first."))
+                msg = localise(msg)+"\n"+localise("Please add some words first.")
             else:
-                drop_to_synthloop = (partials_langs or get_synth_if_possible("en",0) or viable_synths) # the get_synth_if_possible call here is basically to ensure viable_synths is populated
+                drop_to_synthloop = (synth_partials_voices or get_synth_if_possible("en",0) or viable_synths) # the get_synth_if_possible call here is basically to ensure viable_synths is populated
                 msg += "\nPlease read the instructions on the website\nwhich tell you how to add words.\n"+cond(drop_to_synthloop,"Dropping back to justSynthesize loop.\n","")
             if drop_to_synthloop:
                 clearScreen() ; show_info(msg)

gradint-build/src/makeevent.py → src/makeevent.py

-# This file is part of the source code of
-# gradint v0.9962 (c) 2002-2010 Silas S. Brown. GPL v3+.
+# This file is part of the source code of Gradint
+# (c) Silas S. Brown.
 #    This program is free software; you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by
 #    the Free Software Foundation; either version 3 of the License, or
@@ -17,10 +17,11 @@ def filesToEvents(files,dirBase=None):
     # (e.g. when learning poetry, may have a composite
     # prompt)
     if not type(files)==type([]): files = [files]
-    return CompositeEvent(map(lambda x:fileToEvent(x,dirBase),files))
+    return CompositeEvent(map(lambda x,d=dirBase:fileToEvent(x,d),files))
 
 class Partials_Synth(Synth):
     # text is really a list of lists of filenames
+    # TODO if there's a SoundCollector, and if header.wav matches its desired parameters, we can feed bytes directly to it rather than making a temp file; could be useful in environments with Flash disks (NSLU2 etc). (Could have WAV partials there instead, but the resulting number of wav-to-raw sox conversions can slow down SoundCollector on large texts (TODO document this))
     def guess_length(self,lang,text):
         l=(len(text)-1)*0.3
         for phrase in text: l+=quickGuess(len(phrase),1.5) # TODO assumes average of 1.5 partials/sec; needs to be better than that
@@ -39,16 +40,17 @@ class Partials_Synth(Synth):
         # the first syllable in 1st list can optionally be the header file to use
         fname = os.tempnam()+dotwav
         o=open(fname,"wb")
-        if not (text and text[0] and text[0][0].endswith(dotwav)): o.write(open(partialsDirectory+os.sep+"header"+dotwav,"rb").read())
+        if not (text and text[0] and B(text[0][0]).endswith(B(dotwav))): o.write(read(partialsDirectory+os.sep+"header"+dotwav))
         for phrase in text:
-            datFileInUse = 0
+            datFileInUse = 0 ; assert type(phrase)==type([])
             for f in phrase:
-                if f in audioDataPartials:
+                f = S(f)
+                if checkIn(f,audioDataPartials):
                     datFile,offset,size = audioDataPartials[f]
                     if not datFileInUse: datFileInUse = open(partialsDirectory+os.sep+datFile,"rb")
                     datFileInUse.seek(offset) ; o.write(datFileInUse.read(size))
-                else: o.write(open(partialsDirectory+os.sep+f,"rb").read())
-            if not phrase==text[-1]: o.write(chr(0)*partials_raw_0bytes)
+                else: o.write(read(partialsDirectory+os.sep+f))
+            if not phrase==text[-1]: o.write(chr(0)*partials_raw_0bytes) # TODO what if there's a duplicate of text[-1]
         # MUST fix the length, for Windows etc:
         wavLen = o.tell()-8
         o.seek(4) ; o.write(chr(wavLen&0xFF)+chr((wavLen>>8)&0xFF)+chr((wavLen>>16)&0xFF)+chr(wavLen>>24))
@@ -57,19 +59,32 @@ class Partials_Synth(Synth):
 
 def fileToEvent(fname,dirBase=None):
     if dirBase==None: dirBase=samplesDirectory
-    if dirBase: dirBase += os.sep
-    orig_fname = fname
-    if os.sep in fname and fname.find("!synth:")==-1: dirBase,fname = dirBase+fname[:fname.rindex(os.sep)+1], fname[fname.rindex(os.sep)+1:]
-    if "_" in fname: lang = languageof(fname)
+    dirBase,fname = B(dirBase),B(fname)
+    if dirBase: dirBase += B(os.sep)
+    orig_DB,orig_fname = dirBase,fname
+    if B(os.sep) in fname and fname.find(B("!synth:"))==-1: dirBase,fname = dirBase+fname[:fname.rindex(B(os.sep))+1], fname[fname.rindex(B(os.sep))+1:]
+    if B("_") in fname: lang = languageof(fname)
     else: lang="-unknown-" # so can take a simple wav file, e.g. for endAnnouncement
-    if dirBase+fname in variantFiles:
+    if checkIn(dirBase+fname,variantFiles):
         variantFiles[dirBase+fname]=variantFiles[dirBase+fname][1:]+[variantFiles[dirBase+fname][0]] # cycle through the random order of variants
-        fname=variantFiles[dirBase+fname][0]
-    if fname.lower().endswith(dottxt) and "_" in fname: fname = "!synth:"+u8strip(open(dirBase+fname,"rb").read()).strip(wsp)+'_'+lang
-    if fname.find("!synth:")>-1:
+        fname=B(variantFiles[dirBase+fname][0])
+    if fname.lower().endswith(B(dottxt)) and B("_") in fname:
+        ftxt = bwspstrip(u8strip(read(dirBase+fname)))
+        if not ftxt: raise MessageException(B("File ")+fname+B(" in ")+dirBase+B(" has no text in it; please fix this")) # nicer message than catching it at a lower level
+        fname = B("!synth:")+B(ftxt)+B('_')+B(lang)
+    if fname.find(B("!synth:"))>=0:
         s = synthcache_lookup(fname)
         if type(s)==type([]): # trying to synth from partials
-            if partials_raw_mode: e=synth_event(None,s)
+            if filter(lambda x:not type(x)==type([]), s): # but not completely (switching between partials and synth in a long text), this is more tricky:
+                eList = []
+                for phrase in s:
+                    if type(phrase)==type([]):
+                      if partials_raw_mode: eList.append(synth_event(None,[phrase]))
+                      else: eList.append(optimise_partial_playing(CompositeEvent(map(lambda x:SampleEvent(partialsDirectory+os.sep+x,useExactLen=True),phrase))))
+                    else: eList.append(phrase) # it will be a SynthEvent
+                    eList.append(Event(betweenPhrasePause))
+                e = CompositeEvent(eList[:-1]) # omit trailing pause
+            elif partials_raw_mode: e=synth_event(None,s)
             else: e=optimise_partial_playing_list([CompositeEvent(map(lambda x:SampleEvent(partialsDirectory+os.sep+x,useExactLen=True),phrase)) for phrase in s]) # (tell SampleEvent to useExactLen in this case - don't want ANY pause between them)
             if not e: # can't make a ShellEvent from the whole lot; try making individual ShellEvents:
                 e=[]
@@ -78,71 +93,101 @@ def fileToEvent(fname,dirBase=None):
                     e.append(Event(betweenPhrasePause))
                 e=CompositeEvent(e[:-1]) # omit trailing pause
             if not lessonIsTight(): e.length=math.ceil(e.length) # (TODO slight duplication of logic from SampleEvent c'tor)
-        elif s: e=SampleEvent(synthCache+os.sep+s) # single file in synth cache
-        else: e=synth_event(languageof(fname),textof(fname))
-        e.is_prompt=(dirBase==promptsDirectory+os.sep)
+        elif s: e=SampleEvent(synthCache+os.sep+S(s)) # single file in synth cache
+        else:
+            e=synth_event(languageof(fname),textof(fname))
+            e.file = orig_DB+orig_fname # for trace.py check_for_pictures
+        e.is_prompt=(dirBase==B(promptsDirectory+os.sep))
     else: e=SampleEvent(dirBase+fname)
     e.setOnLeaves('wordToCancel',orig_fname)
     return e
 
 transTbl = "TRANS"+extsep+"TBL"
+if mp3web: # synth-cache must exist
+    if not synthCache: synthCache = "synth-cache"
+    try: os.mkdir(synthCache)
+    except: pass # already exists, temporarily-dangling symlink, etc
+
 if synthCache:
     # this listdir() call can take ages on rpcemu if it's large
     if riscos_sound: show_info("Reading synthCache... ")
-    try: synthCache_contents = os.listdir(synthCache)
+    try: synthCache_contents = map(B,os.listdir(synthCache))
     except: synthCache_contents = synthCache = []
     for i in synthCache_contents:
-        if i.upper()==transTbl: # in case it's a different case
-            transTbl=i ; break
+        if i.upper()==B(transTbl): # in case it's a different case
+            transTbl=S(i) ; break
     synthCache_contents = list2dict(synthCache_contents) # NOT 2set, as the GUI can delete things from it
     if riscos_sound: show_info("done\n")
 synthCache_transtbl = {}
-if synthCache and transTbl in synthCache_contents:
+if synthCache and checkIn(B(transTbl),synthCache_contents):
     ensure_nodups = {} # careful of duplicate filenames being appended to trans.tbl, make sure they override previous entries
-    for l in open(synthCache+os.sep+transTbl).readlines():
-        v,k = l.strip(wsp).split(None,1)
-        if v in ensure_nodups: del synthCache_transtbl[ensure_nodups[v]]
+    for l in open(synthCache+os.sep+transTbl,'rb').readlines():
+        v,k = bwspstrip(l).split(None,1)
+        if checkIn(v,ensure_nodups): del synthCache_transtbl[ensure_nodups[v]]
         ensure_nodups[v]=k ; synthCache_transtbl[k]=v
     del ensure_nodups
-def textof(fname): return fname[fname.find('!synth:')+7:fname.rfind('_')]
+def textof(fname):
+    fname = B(fname)
+    return fname[fname.find(B('!synth:'))+7:fname.rfind(B('_'))]
 last_partials_transliteration = None
-synth_partials_cache = {}
+synth_partials_cache = {} ; scl_disable_recursion = 0
 def synthcache_lookup(fname,dirBase=None,printErrors=0,justQueryCache=0,lang=None):
-    # if justQueryCache (used by the GUI), return value is (synthCache_transtbl key, result if any).  If key starts with _, we got a sporadic one.
+    # if justQueryCache (used by the GUI), return value is (synthCache_transtbl key, result if any).  If key starts with _, we got a sporadic one.  (If =2, query sporadic ones but also try partials, used for can_be_synthesized)
     if dirBase==None: dirBase=samplesDirectory
     if dirBase: dirBase += os.sep
     if not lang: lang = languageof(fname)
-    if fname.lower().endswith(dottxt):
-        try: fname = fname[:fname.rfind("_")]+"!synth:"+u8strip(open(dirBase+fname,"rb").read()).strip(wsp)+"_"+lang
+    fname = B(fname)
+    if fname.lower().endswith(B(dottxt)):
+        try: fname = fname[:fname.rfind(B("_"))]+B("!synth:")+bwspstrip(u8strip(read(S(B(dirBase)+B(fname)))))+B("_")+B(lang)
         except IOError: return 0,0 # probably trying to synthcache_lookup a file with variants without first choosing a variant (e.g. in anticipation() to check for sporadic cache entries in old words) - just ignore this
     text = textof(fname)
     useSporadic = -1 # undecided (no point accumulating counters for potentially-unbounded input)
-    if justQueryCache: useSporadic=1
+    if justQueryCache: useSporadic,tryHarder=1,0
+    else: tryHarder = not get_synth_if_possible(lang,0) # (tryHarder = always use sporadic if can't synth from partials, because there's no other synth to fall back on)
     if synthCache:
       for init in "_","":
         for ext in "wav","mp3":
-            k=init+text.lower()+"_"+lang+extsep+ext
-            s=synthCache_transtbl.get(k,k)
-            if s in synthCache_contents: ret=s
-            elif s.lower().endswith(dotwav) and s[:-len(dotwav)]+dotmp3 in synthCache_contents: ret=s[:-len(dotwav)]+dotmp3
+            k=B(init)+text.lower()+B("_"+lang+extsep+ext)
+            s=B(synthCache_transtbl.get(k,k))
+            if checkIn(s,synthCache_contents): ret=s
+            elif s.lower().endswith(B(dotwav)) and checkIn(s[:-len(dotwav)]+B(dotmp3),synthCache_contents): ret=s[:-len(dotwav)]+B(dotmp3)
             else: ret=0
             if ret:
-                if justQueryCache: ret=(k,ret)
+                if justQueryCache==1: ret=(k,ret)
                 if init=="_":
                     if useSporadic==-1: useSporadic=decide_subst_synth(text)
                     if useSporadic: return ret
+                    elif tryHarder: tryHarder=ret
                 else: return ret
-    if justQueryCache: return 0,0
-    if lang not in partials_langs: l,translit=None,None # don't bother trying to transliterate here if there aren't even any partials for that language
-    elif (lang,text) not in synth_partials_cache:
+    if justQueryCache==1: return 0,0
+    if not checkIn(lang,synth_partials_voices): l,translit=None,None # don't bother trying to transliterate here if there aren't even any partials for that language
+    elif not checkIn((lang,text),synth_partials_cache):
         # See if we can transliterate the text first.
         synth,translit = get_synth_if_possible(lang,0,to_transliterate=True),None
-        if synth: translit=synth.transliterate(lang,text)
-        if translit: text=translit
-        if lang=="zh": t2=sort_out_pinyin_3rd_tones(pinyin_uColon_to_V(text)) # need to do this BEFORE stripPuncEtc, for correct sandhi blocking
-        else: t2=text
-        l = [synth_from_partials(phrase,lang) for phrase in stripPuncEtc(t2.lower())] # TODO do we really want to be able to pick new voices at every phrase?  if not, would have to pass the pause points into synth_from_partials itself
-        if None in l: l=None # if any of the partials-phrases failed, fail all (don't mix partials and synth for different parts of the same phrase, it's too confusing)
+        if espeak_language_aliases.get(lang,lang) in ["zhy","zh-yue"]:
+          text2=preprocess_chinese_numbers(fix_compatibility(ensure_unicode(text)),isCant=1).encode('utf-8')
+          if ekho_speed_delta and type(get_synth_if_possible(lang,0))==EkhoSynth: return # don't use partials at all if they've got ekho and have set a different speed (TODO unless they've also changed the speed of the partials? but that would impair the quality advantage of using partials anyway)
+        else: text2=text
+        if synth: translit=synth.transliterate(lang,text2)
+        if translit: t2=translit
+        else: t2=text2
+        if lang=="zh": t2=sort_out_pinyin_3rd_tones(pinyin_uColon_to_V(t2)) # need to do this BEFORE stripPuncEtc, for correct sandhi blocking
+        phraseList = stripPuncEtc(t2.lower())
+        l = [synth_from_partials(phrase,lang) for phrase in phraseList] # TODO do we really want to be able to pick new voices at every phrase?  if not, would have to pass the pause points into synth_from_partials itself
+        if checkIn(None,l): # at least one of the partials-phrases failed
+          global scl_disable_recursion
+          if len(t2)<100 or not filter(lambda x:x,l) or scl_disable_recursion: l=None # don't mix partials and synth for different parts of a short phrase, it's too confusing (TODO make the 100 configurable?)
+          elif type(get_synth_if_possible(lang,0))==EkhoSynth: l=None # some faulty versions of Ekho are more likely to segfault if called on fragments (e.g. if the fragment ends with some English), so don't do this with Ekho (unless can confirm it's at least ekho_4.5-2ubuntu10.04 .. not all versions of ekho can report their version no.)
+          else: # longer text and SOME can be synth'd from partials: go through it more carefully
+            t2=fix_compatibility(ensure_unicode(text2.replace(chr(0),"")).replace(u"\u3002",".").replace(u"\u3001",",")).encode('utf-8')
+            for t in ".!?:;,": t2=t2.replace(B(t),B(t)+chr(0))
+            l=[]
+            scl_disable_recursion = 1
+            for phrase in filter(lambda x:x,t2.split(chr(0))):
+              ll=synthcache_lookup(B("!synth:")+phrase+B("_"+lang),dirBase,0,0,lang)
+              if type(ll)==type([]): l += ll
+              else: l.append(synth_event(lang,phrase,0))
+            scl_disable_recursion = 0
         synth_partials_cache[(lang,text)]=(l,translit)
     else: l,translit=synth_partials_cache[(lang,text)]
     if l and partials_are_sporadic:
@@ -152,27 +197,40 @@ def synthcache_lookup(fname,dirBase=None,printErrors=0,justQueryCache=0,lang=Non
         global last_partials_transliteration
         last_partials_transliteration=translit
     if l: return l
-    if printErrors and synthCache and not (app and winsound): show_info("Not in cache: "+repr(text.lower()+"_"+lang)+"\n",True)
+    if tryHarder and not tryHarder==True: return tryHarder
+    if printErrors and synthCache and not (app and winsound):
+        r = repr(text.lower()+B("_"+lang))
+        if len(r)>100: r=r[:100]+"..."
+        global NICcount
+        try: NICcount += 1
+        except: NICcount=1
+        if NICcount>20: pass
+        elif NICcount==20: show_info("Further 'not in cache' warnings turned off\n",True) # (important on S60 etc; TODO configurable?)
+        else: show_info("Not in cache: "+r+"\n",True)
 def can_be_synthesized(fname,dirBase=None,lang=None):
     if dirBase==None: dirBase=samplesDirectory
     if dirBase: dirBase += os.sep
     if not lang: lang = languageof(fname)
     if get_synth_if_possible(lang,0): return True
-    elif synthcache_lookup(fname,dirBase,1,lang=lang): return True
+    elif synthcache_lookup(fname,dirBase,1,2,lang): return True
     else: return get_synth_if_possible(lang) # and this time print the warning
 def stripPuncEtc(text):
     # For sending text to synth_from_partials.  Removes spaces and punctuation from text, and returns a list of the text split into phrases.
-    for t in " -_'\"()[]": text=text.replace(t,"")
-    for t in ".!?:;": text=text.replace(t,",")
-    return filter(lambda x:x,text.split(","))
+    text = B(text)
+    for t in " -_'\"()[]": text=text.replace(B(t),B(""))
+    for t in ".!?:;": text=text.replace(B(t),B(","))
+    return filter(lambda x:x,text.split(B(",")))
 
-for zipToCheck in ["yali-voice","cameron-voice"]:
+for zipToCheck in ["yali-voice","yali-lower","cameron-voice"]:
     if riscos_sound:
         if fileExists(zipToCheck+"/exe") or fileExists(samplesDirectory+"."+zipToCheck+"/exe") or fileExists(os.getcwd()[:os.getcwd().rindex(".")]+"."+zipToCheck+"/exe"): show_warning("RISC OS users: Please rename the file "+zipToCheck+"/exe to "+zipToCheck+"/zip and unpack it into the gradint directory.")
     elif not winsound: # ok if mingw32, appuifw etc (unzip_and_delete will warn)
         for d in [os.getcwd()+cwd_addSep,".."+os.sep,samplesDirectory+os.sep]:
             f=d+zipToCheck+".exe"
-            if fileExists(f): unzip_and_delete(f,ignore_fail=1) # ignore the error exit status from unzip, which will be because of extra bytes at the beginning
+            if fileExists(f):
+                unzip_and_delete(f,ignore_fail=1) # ignore the error exit status from unzip, which will be because of extra bytes at the beginning
+                try: os.unlink("setup.bat")
+                except: pass
 
 # Filename / Unicode translation - need some safety across filesystems.  synthCache(+utils) could be done this way also rather than having TRANS.TBL (however I'm not sure it would save that much code)
 non_normal_filenames = {} ; using_unicode_filenames=0
@@ -184,7 +242,7 @@ def filename2unicode(f):
     def u8_or_raw(s):
         try: return unicode(s,"utf-8")
         except UnicodeDecodeError: return unicode(s,"latin1") # (actually should try the local codepage on Windows for correct display, but at least this stops a crash)
-    if f.find("_u")>-1 or f.find("_U")>-1:
+    if f.find("_u")>=0 or f.find("_U")>=0:
         try: return unicode(f.replace("_u","\\u").replace("_U","\\u"),"unicode_escape")
         except UnicodeDecodeError: # oops, need to be more careful
             ret = []
@@ -205,7 +263,7 @@ def filename2unicode(f):
     return u
 def unicode2filename(u):
     if using_unicode_filenames: return u
-    if u in non_normal_filenames: return non_normal_filenames[u]
+    if checkIn(u,non_normal_filenames): return non_normal_filenames[u]
     f=u.encode("unicode_escape").replace("\\u","_u")
     for unsafe_char in "?+*<=": f=f.replace(unsafe_char,"_u%04x" % ord(unsafe_char))
     return f
@@ -215,27 +273,35 @@ partials_cache_file="partials-cache"+extsep+"bin"
 partials_language_aliases = {}
 if partialsDirectory and isDirectory(partialsDirectory):
   dirsToStat = []
+  partialsCacheFormat="(partials_raw_mode,synth_partials_voices,guiVoiceOptions,audioDataPartials,dirsToStat,espeak_language_aliases,partials_language_aliases)"
   if pickle and fileExists(partials_cache_file):
     try:
-        partials_langs,partials_raw_mode,synth_partials_voices,audioDataPartials,dirsToStat,ela,partials_language_aliases = pickle.Unpickler(open(partials_cache_file,"rb")).load()
-        if not ela==espeak_language_aliases: dirsToStat=[]
-        del ela
+        ela = espeak_language_aliases
+        format,values = pickle.Unpickler(open(partials_cache_file,"rb")).load()
+        if format==partialsCacheFormat: exec (format+"=values")
+        if not (ela==espeak_language_aliases and dirsToStat[0][0]==partialsDirectory): espeak_language_aliases,dirsToStat=ela,[]
+        del ela,format,values
+    except MemoryError: raise # has been known on winCEsound when we're a library module (so previous memory check didn't happen)
     except: dirsToStat = []
     for d,result in dirsToStat:
       if not tuple(os.stat(d))==result:
         dirsToStat=[] ; break
   if not dirsToStat: # need to re-scan
     if riscos_sound or winCEsound: show_info("Scanning partials... ")
-    partials_langs = os.listdir(partialsDirectory)
+    guiVoiceOptions = []
+    langs = os.listdir(partialsDirectory)
     dirsToStat.append((partialsDirectory,os.stat(partialsDirectory)))
-    audioDataPartials = {}
-    partials_raw_mode = "header"+dotwav in partials_langs
-    for l in partials_langs:
+    audioDataPartials = {} ; synth_partials_voices = {}
+    partials_raw_mode = checkIn("header"+dotwav,langs)
+    for l in langs:
         try: voices = os.listdir(partialsDirectory+os.sep+l)
         except: voices = []
         if voices: dirsToStat.append((partialsDirectory+os.sep+l,os.stat(partialsDirectory+os.sep+l)))
         thisLangVoices = [] ; voices.sort()
         for v in voices:
+            if "-" in v and v[:v.index("-")] in voices:
+              suffix=v[v.index("-"):]
+              if not checkIn(suffix,guiVoiceOptions): guiVoiceOptions.append(suffix)
             start,mid,end = [],[],[] ; flags=0
             try: files = os.listdir(partialsDirectory+os.sep+l+os.sep+v)
             except: files = []
@@ -255,10 +321,11 @@ if partialsDirectory and isDirectory(partialsDirectory):
                     while True:
                         fftell = ff.tell()
                         char = ff.read(1)
-                        if not "0"<=char<="9": break
-                        size,fname = (char+ff.readline(256)).strip(wsp).split(None,1)
+                        if not B("0")<=char<=B("9"): break
+                        size,fname = bwspstrip(char+ff.readline(256)).split(None,1)
                         try: size=int(size)
                         except: break # binary just happened to start with "0"-"9"
+                        fname = S(fname)
                         addFile(fname)
                         amend.append(l+os.sep+v+os.sep+fname)
                         audioDataPartials[l+os.sep+v+os.sep+fname] = (f,offset,size)
@@ -268,39 +335,41 @@ if partialsDirectory and isDirectory(partialsDirectory):
                 if partials_raw_mode:
                     if not f.endswith(extsep+"raw"): return
                 elif not f.endswith(dotwav) or f.endswith(dotmp3): return
-                if f.find("-s")>-1 or f.find("-i")>-1: start.append(f) # 'start' or 'initial'
-                elif not "-" in f or f.find('-m')>-1: mid.append(f)
-                elif f.find('-e')>-1 or f.find('-f')>-1: end.append(f) # 'end' or 'finish'
+                if f.find("-s")>=0 or f.find("-i")>=0: start.append(f) # 'start' or 'initial'
+                elif not "-" in f or f.find('-m')>=0: mid.append(f)
+                elif f.find('-e')>=0 or f.find('-f')>=0: end.append(f) # 'end' or 'finish'
             for f in files: addFile(f)
             def byReverseLength(a,b): return len(b)-len(a)
-            start.sort(byReverseLength) ; mid.sort(byReverseLength) ; end.sort(byReverseLength) # important if there are some files covering multiple syllables (and do it to start,mid,end not to files initially, so as to catch files within audiodata.dat also)
+            sort(start,byReverseLength) ; sort(mid,byReverseLength) ; sort(end,byReverseLength) # important if there are some files covering multiple syllables (and do it to start,mid,end not to files initially, so as to catch files within audiodata.dat also)
             def toDict(l): # make the list of filenames into a dict of short-key -> [(long-key, filename) list].  short-key is the shortest possible key.
                 if not l: return {}
                 l2 = [] ; kLen = len(l[0])
                 for i in l:
                     if "-" in i: key=i[:i.index("-")]
                     else: key=i[:i.rindex(extsep)]
-                    if key.find("_u")>-1 or key.find("_U")>-1: # a unicode partial with a portable filename?
+                    if key.find("_u")>=0 or key.find("_U")>=0: # a unicode partial with a portable filename?
                         key = filename2unicode(key).encode('utf-8')
                     l2.append((key,i))
                     kLen=min(kLen,len(key))
                 l = {}
                 for k,i in l2:
-                    if not k[:kLen] in l: l[k[:kLen]]=[]
+                    if not checkIn(k[:kLen],l): l[k[:kLen]]=[]
                     l[k[:kLen]].append((k,i))
                 return l
             thisLangVoices.append((v,toDict(start),toDict(mid),toDict(end),flags))
         synth_partials_voices[l] = thisLangVoices
-        if l in espeak_language_aliases: partials_language_aliases[espeak_language_aliases[l]]=l
+        if checkIn(l,espeak_language_aliases): partials_language_aliases[espeak_language_aliases[l]]=l
     if riscos_sound or winCEsound: show_info("done\n")
     if pickle:
-      try: pickle.Pickler(open(partials_cache_file,"wb"),-1).dump((partials_langs,partials_raw_mode,synth_partials_voices,audioDataPartials,dirsToStat,espeak_language_aliases,partials_language_aliases))
+      try: pickle.Pickler(open(partials_cache_file,"wb"),-1).dump((partialsCacheFormat,eval(partialsCacheFormat)))
       except IOError: pass # ignore write errors as it's only a cache
       except OSError: pass
   if partials_raw_mode:
-    (wtype,wrate,wchannels,wframes,wbits) = sndhdr.what(partialsDirectory+os.sep+"header"+dotwav)
-    partials_raw_0bytes = int(betweenPhrasePause*wrate)*wchannels*(wbits/8)
-else: partials_langs,partials_raw_mode = [],None
+    (wtype,wrate,wchannels,wframes,wbits) = swhat(partialsDirectory+os.sep+"header"+dotwav)
+    partials_raw_0bytes = int(betweenPhrasePause*wrate)*wchannels*int(wbits/8)
+else: synth_partials_voices,partials_raw_mode = {},None
+
+if checkIn("cant",synth_partials_voices): synth_partials_voices["zhy"]=synth_partials_voices["zh-yue"]=synth_partials_voices["cant"]
 
 def partials_langname(lang):
     lang = espeak_language_aliases.get(lang,lang)
@@ -309,41 +378,50 @@ def partials_langname(lang):
 
 def synth_from_partials(text,lang,voice=None,isStart=1):
     lang = partials_langname(lang)
-    text=text.strip(wsp) # so whitespace between words is ignored on the recursive call
+    text=bwspstrip(B(text)) # so whitespace between words is ignored on the recursive call
     if lang=="zh": # hack for Mandarin - higher tone 5 after a tone 3 (and ma5 after 4 or 5 also)
         lastNum = None
         for i in range(len(text)):
-            if text[i] in "123456":
-                if text[i]=="5" and (lastNum=="3" or (lastNum>"3" and i>2 and text[i-2:i+1]=="ma5")): # (TODO ne5 also? but only if followed by some form of question mark, and that might have been dropped)
+            if text[i:i+1] in B("123456"):
+                if text[i:i+1]==B("5") and (lastNum==B("3") or (lastNum and lastNum>B("3") and i>2 and text[i-2:i+1]==B("ma5"))): # (TODO ne5 also? but only if followed by some form of question mark, and that might have been dropped)
                     # see if we got a "tone 6" (higher tone 5)
                     # don't worry too much if we haven't
-                    r=synth_from_partials(text[:i]+"6"+text[i+1:],lang,voice,isStart)
+                    r=synth_from_partials(text[:i]+B("6")+text[i+1:],lang,voice,isStart)
                     if r: return r
                     else: break
                 elif lastNum: break # don't look beyond 1st 2
-                lastNum = text[i]
+                lastNum = text[i:i+1]
     if not voice: # try all voices for the language, see if we can find one that can say all the necessary parts
-        if not lang in synth_partials_voices: return None
+        if not checkIn(lang,synth_partials_voices): return None
         needCalibrated=False
         if lang=="zh": # hack for Mandarin - avoid consecutive 1st tones on non-calibrated voices
             # (DON'T do 3rd tone sandhi here - that's the caller's responsibility and we don't want it done twice now there's sandhi-blocking rules)
             lastNum=None
-            for c in text:
-                if c=="1" and lastNum=="1":
+            for i in xrange(len(text)):
+                c = text[i:i+1]
+                if c==B("1") and lastNum==B("1"):
                     needCalibrated=True ; break # TODO: unless this syllable is exactly the same as the last syllable (a repeated syllable is always ok to use even if uncalibrated)
-                if c in "123456": lastNum=c
+                if c in B("123456"): lastNum=c
             # end of hack for Mandarin
-        for v in synth_partials_voices[lang]:
+        vTry = synth_partials_voices[lang]
+        if voiceOption:
+            vt1=[] ; vt2=[]
+            for v in vTry:
+              if v[0].endswith(voiceOption): vt1.append(v)
+              else: vt2.append(v)
+            vTry=vt1+vt2
+        for v in vTry:
             if needCalibrated and not v[-1]&1: continue
             r = synth_from_partials(text,lang,v)
             if r:
                 if partials_raw_mode and v[-1]&2: r.insert(0,"header"+dotwav)
-                return map(lambda x:lang+os.sep+v[0]+os.sep+x,r)
+                return map(lambda x,v=v,lang=lang:lang+os.sep+v[0]+os.sep+x,r)
         return None
     dir, start, mid, end, flags = voice
     def lookup_dic(text,dic):
+        text = S(text)
         if dic:
-            for k,v in dic.get(text[:len(dic.keys()[0])],[]):
+            for k,v in dic.get(text[:len(list(dic.keys())[0])],[]):
                 if text.startswith(k): return k,v
         return None,None
     if not text: return [] # (shouldn't happen)
@@ -374,12 +452,12 @@ def optimise_partial_playing(ce):
     for e in ce.eventList[1:]:
         if not soundFileType(e.file)==fileType: return ce # must be all the same type for this optimisation
     s = None
-    if fileType=="mp3" and madplay_program and not macsound and not cygwin: # (don't do this on cygwin because cygwin will require changeToDirOf and that could get awkward)
+    if fileType=="mp3" and madplay_path and mp3Player==madplay_path and not macsound and not cygwin: # (don't do this on cygwin because cygwin will require changeToDirOf and that could get awkward)
         # mp3 probably has encoding gaps etc, but we can try our best
-        if playProgram=="aplay": s=ShellEvent(madplay_program+' -q -A $Vol$'+''.join(map(lambda x:' "'+x.file+'"', ce.eventList))+' -o wav:-|aplay -q',True) # (set retryOnFail=True)
-        else: s=ShellEvent(madplay_program+' -q -A $Vol$'+''.join(map(lambda x:' "'+x.file+'"', ce.eventList)),True)
+        if wavPlayer=="aplay": s=ShellEvent(mp3Player+' -q -A $Vol$'+''.join(map(lambda x:' "'+x.file+'"', ce.eventList))+' -o wav:-|aplay -q',True) # (set retryOnFail=True)
+        else: s=ShellEvent(mp3Player+' -q -A $Vol$'+''.join(map(lambda x:' "'+x.file+'"', ce.eventList)),True)
         s.VolReplace="soundVolume_dB"
-    elif (not fileType=="mp3") and playProgram in ["aplay","sox"]:
+    elif (not fileType=="mp3") and (wavPlayer in ["aplay","sox"] or wavPlayer.strip().endswith("<")):
         # if they're all the same format, we can use sox concatenation (raw, with an unspecified-length wav header at start)
         # (don't try to do that if different formats - the low-end hardware may not take the rate conversion)
         ok=gotSox
@@ -389,21 +467,25 @@ def optimise_partial_playing(ce):
                 if not simplified_header(e.file)==format:
                     ok=False ; break
         if ok:
-            s=ShellEvent('set -o pipefail;('+'&&'.join(['cat "%s" | sox -t %s - -t wav - $Vol$ 2>/dev/null' % (ce.eventList[0].file,fileType)]+['cat "%s" | sox -t %s - -t raw - $Vol$'%(e.file,fileType) for e in ce.eventList[1:]])+')|'+cond(playProgram=="aplay",'aplay -q','sox -t wav - '+sox_type+' '+oss_sound_device),True)
+            if wavPlayer=="aplay": wpMod="aplay -q"
+            elif wavPlayer.strip().endswith("<"): wpMod=wavPlayer.strip()[:-1] # nc etc
+            else: wpMod='sox -t wav - '+sox_type+' '+oss_sound_device
+            s=ShellEvent('set -o pipefail;('+'&&'.join(['cat "%s" | sox -t %s - -t wav - $Vol$ 2>/dev/null' % (S(ce.eventList[0].file),fileType)]+['cat "%s" | sox -t %s - -t raw - $Vol$'%(S(e.file),fileType) for e in ce.eventList[1:]])+')'+sox_ignoreLen+'|'+wpMod,True)
             s.VolReplace="sox_effect"
-        elif playProgram=="aplay" and not sox_effect: s=ShellEvent('aplay -q '+''.join(map(lambda x:' "'+x.file+'"', ce.eventList)),True) # (which is not quite as good but is the next best thing) (and hope they don't then try to re-play a saved lesson with a volume adjustment)
+        elif wavPlayer=="aplay" and not sox_effect: s=ShellEvent('aplay -q '+''.join(map(lambda x:' "'+x.file+'"', ce.eventList)),True) # (which is not quite as good but is the next best thing) (and hope they don't then try to re-play a saved lesson with a volume adjustment)
     if s:
         s.length = ce.length
         s.equivalent_event_list = ce.eventList
         return s
     else: return ce # can't figure out an optimisation in these circumstances
 def simplified_header(fname):
-    h=sndhdr.what(fname)
+    # called by optimise_partial_playing(_list)
+    h=swhat(fname)
     # ignore num frames i.e. h[3], just compare formats
     if h: return h[:3]+h[4:]
 def optimise_partial_playing_list(ceList):
     # similar to above, but returns a ShellEvent for a list of ce's that are to be separated by short pauses, or None if can't do this optimisation.  This is because sox on NSLU2's etc has too much latency for the short pauses.
-    if (soundCollector and not saveLesson) or not playProgram=="aplay" or not gotSox: return
+    if (soundCollector and not saveLesson) or not wavPlayer=="aplay" or not gotSox: return
     format = None ; l = [] ; theLen = 0
     for ce in ceList:
         for e in ce.eventList:
@@ -416,7 +498,7 @@ def optimise_partial_playing_list(ceList):
         if not ce==ceList[-1]:
             l.append('dd if=/dev/zero bs=%d count=1 2>/dev/null' % (int(betweenPhrasePause*format[1])*format[2]*(format[-1]/8)))
             theLen += betweenPhrasePause
-    s=ShellEvent('('+';'.join(l)+')|aplay -q',True)
+    s=ShellEvent('('+';'.join(l)+')'+sox_ignoreLen+'|aplay -q',True) # sox_ignoreLen is needed for newer 'sox' versions which manage to write the 1st syllable's length no matter how we try to stop it
     s.length = theLen
     s.VolReplace="sox_effect"
     s.equivalent_event_list = []