gradint-build/samples/utils/list2cache.py → samples/utils/list2cache.py

-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 # list2cache.py language
 

gradint-build/samples/utils/manual-splitter.py → samples/utils/manual-splitter.py

 #!/usr/bin/env python
 
-# Like splitter.py, but lets you use Audacity etc to split in non-realtime.
+# Like Gradint's "record from file", but lets you use Audacity etc to split in non-realtime.
 # You must export the segments in order.
 
 # ('mv && increment count' in a loop: ok as long as in same dir so no complicatns w cross-device & still-open)

samples/utils/player.py

+#!/usr/bin/env python
+# (should work in both Python 2 and Python 3)
+
+# Simple sound-playing server v1.59
+# Silas S. Brown - public domain - no warranty
+
+# connect to port 8124 (assumes behind firewall)
+# and each connection can send WAV or MP3 data
+
+# so gradint advanced.txt can do
+# wavPlayer = mp3Player = "nc HostName 8124 -q 0 <"
+# (most of this script assumes GNU/Linux)
+
+import socket, select, os, sys, os.path, time, re
+for a in sys.argv[1:]:
+  if a.startswith("--rpi-bluetooth-setup"): # tested on Raspberry Pi 400 with OS versions 11 and 12; also tested on Raspberry Pi Zero W with Raspbian 10 Lite (with the device already paired: needed to say "scan on", "discovery on", remove + pair in bluetoothctl).  Send Eth=(bluetooth Ethernet addr) to start.  Note that the setup command reboots the system.
+    os.system(r'if ! grep "$(cat ~/.ssh/*.pub)" ~/.ssh/authorized_keys; then cat ~/.ssh/*.pub >> ~/.ssh/authorized_keys;fi && (echo "[Unit]";echo "Description=Gradint player utility";echo "[Service]";echo "Type=oneshot";echo "ExecStart=bash -c \"while ! ssh localhost true; do sleep 1; done; ssh localhost '+os.path.join(os.getcwd(),sys.argv[0])+r'\"";echo "WorkingDirectory='+os.getcwd()+'";echo User="$(whoami)";echo "[Install]";echo "WantedBy=multi-user.target") > player.service && sudo mv player.service /etc/systemd/system/ && sudo systemctl daemon-reload && sudo systemctl enable player && chmod +x '+sys.argv[0]+' && sudo bash -c "apt-get -y install sox mpg123 pulseaudio pulseaudio-module-bluetooth && usermod -G bluetooth -a $USER && (echo load-module module-switch-on-connect;echo load-module module-bluetooth-policy;echo load-module module-bluetooth-discover) >> /etc/pulse/default.pa && (echo [General];echo FastConnectable = true) >> /etc/bluetooth/main.conf && reboot"') # (eee off: improves reliability of gigabit ethernet on RPi400)
+  elif a=="--aplay": use_aplay = True # aplay and madplay, for older embedded devices, NOT tested together with --rpi-bluetooth-* above
+  elif a.startswith("--delegate="): delegate_to_check=a.split('=')[1] # will ping that IP and delegate all sound to it when it's up.  E.g. if it has better amplification but it's not always switched on.
+  elif a.startswith("--chime="): chime_mp3=a.split('=')[1] # if clock bell desired, e.g. echo '$i-14vfff$c48o0l1b- @'|mwr2ly > chime.ly && lilypond chime.ly && timidity -Ow chime.midi && audacity chime.wav (amplify + trim) + mp3-encode (keep default 44100 sample rate so ~38 frames per sec).  Not designed to work with --delegate.  Pi1's 3.5mm o/p doesn't sound very good with this bell.
+  else: assert 0, "unknown option "+a
+
+os.environ["PATH"] += ":/usr/local/bin"
+try: use_aplay
+except: use_aplay = False
+try: delegate_to_check
+except: delegate_to_check = None
+try: chime_mp3
+except: chime_mp3 = None
+last_chime = last_play = 0
+delegate_known_down = 0
+s=socket.socket()
+s.bind(('',8124))
+s.listen(5)
+if type(b"")==type(""): S=lambda x:x # Python 2
+else: S=lambda x:x.decode("latin1") # Python 3
+eth = ""
+while True:
+    if chime_mp3:
+        t = time.time()
+        if t > last_chime+60 and t%1800 < 60 and not t<last_play+20:
+            last_chime = t ; h,m=time.localtime(t)[3:5]
+            if m>1: numChimes = 1
+            elif not h%12: numChimes = 12
+            else: numChimes = h%12
+            if not 7<=h%24<=22: pass # silence the chime at night
+            elif use_aplay:
+              if numChimes > 1: os.system("(madplay -Q -t 1 -o wav:- '"+chime_mp3+"'"+(";madplay -Q -t 1 -o raw:- '"+chime_mp3+"'")*(numChimes-2)+";madplay -Q -o raw:- '"+chime_mp3+"') | aplay -q")
+              else: os.system("madplay -Q -o wav:- '%s' | aplay -q" % chime_mp3)
+            elif numChimes > 1: os.system("(mpg123 -w - -n 38 --loop %d '%s' ; mpg123 -s '%s') 2>/dev/null | play -t wav --ignore-length - 2>/dev/null" % (numChimes-1,chime_mp3,chime_mp3))
+            else: os.system("mpg123 -q '%s'" % chime_mp3)
+        if not select.select([s],[],[],1800-time.time()%1800)[0]: continue
+    c,(a,port) = s.accept()
+    c.settimeout(10)
+    try: d = S(c.recv(4))
+    except: # e.g. timeout, or there was an error reading the file on the remote side and we got 0 bytes
+        c.close() ; continue
+    if delegate_to_check and not a==delegate_to_check and delegate_known_down < time.time()-60 and not os.system("ping -c 1 -w 0.5 '"+delegate_to_check+"' >/dev/null 2>/dev/null"): player = "nc -N '"+delegate_to_check+"' 8124"
+    elif d=='RIFF': # WAV
+        if use_aplay: player = "aplay -q"
+        else: player = "play - 2>/dev/null"
+    elif d=='STOP':
+        c.close()
+        while not d=='START':
+            c,a = s.accept()
+            try: d = S(c.recv(5))
+            except: d = ""
+            c.close()
+        continue
+    elif d=='QUIT':
+        s.close() ; break
+    elif d=="Eth=": # Eth=ethernet address to connect via Bluetooth (see --rpi-bluetooth-setup above)
+        eth = S(c.recv(17))
+        assert re.match("^[A-Fa-f0-9:]+$",eth)
+        os.system("E="+eth+";if ! pacmd list-sinks | grep "+eth.replace(":","_")+" >/dev/null; then while true; do bluetoothctl --timeout 1 disconnect | grep Missing >/dev/null||sleep 5;T=5;while ! bluetoothctl --timeout $T connect $E | egrep \"Connection successful|Device $E Connected: yes\"; do sleep 5; T=10;bluetoothctl --timeout 1 devices;echo Retrying $E; done ; Got=0; for Try in 1 2 3 4 5 6 7 8 9 a b c d e f g h i j k l m n o p q r s t u v w x y z; do if pacmd list-sinks | grep "+eth.replace(":","_")+" >/dev/null; then Got=1; break; fi; sleep 1; done; if [ $Got = 1 ] ; then break; fi; done; fi; pacmd set-default-sink bluez_sink."+eth.replace(":","_")+".a2dp_sink") # ; play /usr/share/scratch/Media/Sounds/Animal/Dog1.wav # (not really necessary if using 'close the socket' to signal we're ready)
+        c.close() ; continue
+    elif d=="Eth0":
+      if eth: os.system("bluetoothctl --timeout 1 disconnect "+eth)
+      c.close() ; continue
+    elif use_aplay: player = "madplay -Q -o wav:- - | aplay -q" # MP3
+    else: player = "mpg123 - 2>/dev/null" # MP3 non-aplay
+    if delegate_known_down < time.time()-60 and not player.startswith("nc -N "): delegate_known_down = time.time()
+    player = os.popen(player,"w")
+    if type(d)==type(u""): d = d.encode("latin1")
+    while d:
+        try:
+            try: player.write(d)
+            except TypeError: # Python 3
+                player.buffer.write(d)
+        except IOError: break # it was probably killed
+        try: d = c.recv(4096)
+        except: d = ""
+    try:
+        c.close() ; player.close()
+    except: pass
+    last_play = time.time()

samples/utils/recover-unavail.py

+#!/usr/bin/env python2
+
+# Script to recover vocabulary from the "unavailable"
+# entries in Gradint's progress file.  Use if for some
+# reason the vocab file has been truncated (e.g. filesystem
+# problems) and this propagated to your backup system before
+# you noticed.
+
+# v1.0 (c) 2012 Silas S. Brown.  License: GPL
+
+ignore_words_that_are_also_in_backup_unavail = True # if the fault just happened
+
+import gradint, time
+gradint.availablePrompts = gradint.AvailablePrompts()
+d = gradint.ProgressDatabase()
+
+if ignore_words_that_are_also_in_backup_unavail:
+    gradint.progressFile = gradint.progressFileBackup
+    gradint.pickledProgressFile = None
+    d2 = gradint.ProgressDatabase(alsoScan=0)
+    for x in d2.unavail: d.unavail.remove(x)
+
+print "# Words recovered %d-%02d-%02d" % time.localtime()[:3]
+print "# - capitalisation and comments are missing; order may be approximate"
+
+gradint.reallyKnownThreshold = 0
+poems,line2index = gradint.find_known_poems(d.unavail)
+output = [] ; doneAlready = {}
+for pLines in poems:
+    if filter(lambda x:not x.startswith("!synth:") or not gradint.languageof(x)==gradint.secondLanguage, pLines): continue
+    plines2 = []
+    for p in pLines:
+        idx = line2index[p] ; doneAlready[idx] = 1
+        prompt = d.unavail[idx][1]
+        equals = ""
+        if type(prompt)==type([]):
+            if len(prompt)==3: equals = prompt[1]
+        elif not plines2 and not prompt==p: equals=prompt # if 1st line
+        if equals:
+            assert equals.startswith("!synth:") and gradint.languageof(equals)==gradint.firstLanguage, "recovery of poems with non-L1 secondary prompts not yet supported"
+            equals = "="+gradint.textof(equals)
+        plines2.append(gradint.textof(p)+equals)
+    output.append((d.unavail[line2index[pLines[0]]][0], gradint.secondLanguage, gradint.firstLanguage, "\n".join(["begin poetry"]+plines2+["end poetry"])))
+
+for count,(num,L1,L2) in zip(xrange(len(d.unavail)),d.unavail):
+    if count in doneAlready: continue
+    if type(L1)==type(L2)==type("") and L1.startswith("!synth:") and L2.startswith("!synth:"):
+        lang1,lang2 = gradint.languageof(L1),gradint.languageof(L2)
+        output.append((num,lang2,lang1,"%s=%s" % (gradint.textof(L2),gradint.textof(L1))))
+
+output.sort() ; output.reverse()
+curL2,curL1 = None,None
+for num,lang2,lang1,text in output:
+    if not (lang2,lang1) == (curL2,curL1):
+        curL2,curL1 = lang2,lang1
+        print "SET LANGUAGES %s %s" % (curL2,curL1)
+    print text

gradint-build/samples/utils/synth-batchconvert-helper.py → samples/utils/synth-batchconvert-helper.py

-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 # Script to assist with using TextAloud or similar program
 # that can batch-synthesize a collection of text files
@@ -8,7 +8,7 @@
 # words in vocab.txt etc, and rename the resulting *.mp3 or *.wav
 # files into the synth cache.
 
-# Should be useful if you are on Linux and want to run a
+# Should be useful if you are not on Windows and want to run a
 # non-English speech synth in the Windows Emulator (since
 # ptts can have trouble, but tools like TextAloud still work).
 # Note: This script currently assumes that the filesystem
@@ -25,6 +25,12 @@ hanziOnly = 1  # 1 or 0.  If 1 then only phrases consisting
 # entirely of Chinese characters will be listed (could be useful
 # for voices like MeiLing which can't really manage anything else)
 
+# (Note: If you need to artificially specify a
+# division between two hanzi words, use a hyphen
+# (-) to do it.  MeiLing and Gradint/Yali will
+# both recognise this as a word boundary that is
+# not to be pronounced.)
+
 newStuff = "new-stuff" # the directory in which *.txt files
 # will be created, and to look for the resulting *.mp3/*.wav files
 
@@ -37,6 +43,14 @@ delete_old = 1  # if 1 (and if sporadic) then older cached
 # this script moves them there, as that's how it identifies its
 # "own" mp3/wav files (as opposed to anything else you may have cached).
 
+actually_generate = 0 # if 1, will call gradint to generate
+# the cached sound using its choice of voice for that language,
+# instead of relying on your use of TextAloud etc.
+# Might be useful if you need to move it to another machine that
+# doesn't have that voice, and you still want to use sporadic
+# etc (like a more advanced version of cache-synth.py)
+testMode = 0 # if 1 and actually_generate is 1, will play too
+
 # -----------------------------------------
 
 import sys,os,time
@@ -49,60 +63,73 @@ except: pass
 
 sys.argv = []
 import gradint
-gradint.cache_maintenance_mode = 1
+from gradint import dottxt,dotwav,dotmp3
 assert gradint.synthCache, "need a synthCache for this to work"
+gradint.cache_maintenance_mode = 1
 try: trans = open(gradint.synthCache+os.sep+gradint.transTbl).read().replace("\n"," ")+" "
 except: trans = ""
+scld=gradint.list2dict(os.listdir(gradint.synthCache))
 def synth_fileExists(f):
-    try:
-        open(gradint.synthCache+os.sep+f)
-        return 1
-    except: return (" "+f+" ") in trans
+    if f in scld: return True
+    else: return (" "+f+" ") in trans
 
 # Check for previous newStuff .txt's, and any results from them
 generating = {}
 fname2txt = {}
 for l in os.listdir(newStuff):
-    if l.endswith(gradint.dottxt) and "_" in l:
+    if l.endswith(dottxt) and "_" in l:
         txt = open(newStuff+os.sep+l).read().decode('utf-16')
         txt = (sporadic+txt,l[l.rindex("_")+1:l.rindex(gradint.extsep)])
-        generating[txt] = 1 ; fname2txt[l[:l.rindex(gradint.extsep)]]=txt
+        generating[txt] = (None,l)
+        fname2txt[l[:l.rindex(gradint.extsep)]]=txt
 for l in os.listdir(newStuff):
-    if l.endswith(gradint.dotwav) or l.endswith(gradint.dotmp3):
+    if l.endswith(dotwav) or l.endswith(dotmp3):
         k=l[:l.rindex(gradint.extsep)]
         if k in fname2txt: generating[fname2txt[k]]=newStuff+os.sep+l
-del fname2txt # now 'generating' maps (txt,lang) to 1 or filename
+del fname2txt # now 'generating' maps (txt,lang) to (None,txtFile) or filename
+for k,v in generating.items():
+    if type(v)==tuple and v[0]==None: # a previous run was interrupted
+        os.remove(newStuff+os.sep+v[1])
+        del generating[k]
 
 def getTxtLang(s):
     if '!synth:' in s and "_" in s: return gradint.textof(s).decode('utf-8'),gradint.languageof(s)
-    elif s.endswith(gradint.extsep+"txt"): return gradint.readText(s).decode('utf-8'), gradint.languageof(s)
-    else: return None,None
+    elif s.endswith(gradint.extsep+"txt"):
+        langToSynth = gradint.languageof(s)
+        if langToSynth==languageToCache: return gradint.readText(s).decode('utf-8'), langToSynth # else don't bother reading the file (it might be over ftpfs)
+    return None,None
 
 def decache(s):
     textToSynth,langToSynth = getTxtLang(s)
     if not textToSynth: return
     textToSynth="_"+textToSynth # sporadic mode
     generating[(textToSynth.lower(),langToSynth)]=1 # don't re-generate it
-    textToSynth=textToSynth.encode('utf-8')
-    s=textToSynth.lower()+"_"+langToSynth
-    if delete_old and langToSynth==languageToCache and gradint.fileExists_stat(gradint.synthCache+os.sep+s+gradint.dottxt):
-        for ext in [gradint.dottxt,gradint.dotwav,gradint.dotmp3]:
-            try: os.remove(gradint.synthCache+os.sep+s+ext)
-            except: pass
+    s=textToSynth.lower().encode('utf-8')+"_"+langToSynth
+    if delete_old and langToSynth==languageToCache:
+        for ext in [dottxt,dotwav,dotmp3]:
+            if s+ext in scld:
+                os.remove(gradint.synthCache+os.sep+s+ext)
+                del scld[s+ext]
 
 samples = gradint.scanSamples() # MUST call before sporadic so variantFiles is populated
 
 if sporadic:
+  pd = gradint.ProgressDatabase()
   if delete_old: print "Checking for old words to remove"
   else: print "Sporadic mode: Checking for old words to avoid"
-  for t,prompt,target in gradint.ProgressDatabase().data:
+  for t,prompt,target in pd.data:
     if t>=gradint.reallyKnownThreshold:
         if type(prompt)==type([]):
             for p in prompt: decache(p)
         else: decache(prompt)
         decache(target)
 
-count = 0
+count = 0 ; toMove = []
+
+def rename(old,new):
+    # don't use os.rename - can get problems cross-device
+    open(new,"wb").write(open(old,"rb").read())
+    os.remove(old)
 
 def maybe_cache(s):
     textToSynth,langToSynth = getTxtLang(s)
@@ -110,30 +137,59 @@ def maybe_cache(s):
     if not langToSynth==languageToCache: return
     if hanziOnly and not gradint.fix_compatibility(textToSynth).replace(" ","")==gradint.hanzi_and_punc(textToSynth).replace(" ",""): return
     for txt in [textToSynth, sporadic+textToSynth]:
-      if synth_fileExists((txt.encode('utf-8')+"_"+langToSynth+gradint.dotwav).lower()) or synth_fileExists((txt.encode('utf-8')+"_"+langToSynth+gradint.dotmp3).lower()): return # it's already been done
-      if synth_fileExists(("__rejected_"+txt.encode('utf-8')+"_"+langToSynth+gradint.dotwav).lower()) or synth_fileExists(("__rejected_"+txt.encode('utf-8')+"_"+langToSynth+gradint.dotmp3).lower()): return # it's been rejected
+      if synth_fileExists((txt.encode('utf-8')+"_"+langToSynth+dotwav).lower()) or synth_fileExists((txt.encode('utf-8')+"_"+langToSynth+dotmp3).lower()): return # it's already been done
+      if synth_fileExists(("__rejected_"+txt.encode('utf-8')+"_"+langToSynth+dotwav).lower()) or synth_fileExists(("__rejected_"+txt.encode('utf-8')+"_"+langToSynth+dotmp3).lower()): return # it's been rejected
     textToSynth=sporadic+textToSynth
     k = (textToSynth.lower(),langToSynth)
     if generating.has_key(k):
         if not generating[k]==1: # a file already exists
-            # don't use os.rename - can get problems cross-device
-            open(gradint.synthCache+os.sep+textToSynth.encode('utf-8')+'_'+langToSynth+generating[k][generating[k].rindex(gradint.extsep):],"wb").write(open(generating[k],"rb").read())
-            #open(gradint.synthCache+os.sep+textToSynth.encode('utf-8')+'_'+langToSynth+gradint.dottxt,"wb").write(open(generating[k][:generating[k].rindex(gradint.extsep)]+gradint.dottxt,"rb").read())
-            os.remove(generating[k])
-            os.remove(generating[k][:generating[k].rindex(gradint.extsep)]+gradint.dottxt)
+            fname = textToSynth.lower().encode('utf-8')+'_'+langToSynth+generating[k][generating[k].rindex(gradint.extsep):]
+            rename(generating[k],gradint.synthCache+os.sep+fname)
+            scld[fname] = 1
+            #rename(generating[k][:generating[k].rindex(gradint.extsep)]+dottxt,gradint.synthCache+os.sep+textToSynth.lower().encode('utf-8')+'_'+langToSynth+dottxt)
+            os.remove(generating[k][:generating[k].rindex(gradint.extsep)]+dottxt)
             generating[k]=1
         return
+    if actually_generate:
+        tm = [gradint.synth_event(langToSynth,textToSynth[len(sporadic):].encode('utf-8')).getSound(),(textToSynth.encode('utf-8')+"_"+langToSynth+dotwav).lower()]
+        if gradint.got_program("lame"):
+            # we can MP3-encode it (TODO make this optional)
+            n = tm[0][:-len(dotwav)]+dotmp3
+            if not os.system("lame --cbr -h -b 48 -m m \"%s\" \"%s\"" % (tm[0],n)):
+              os.remove(tm[0])
+              tm[0] = n
+              tm[1] = tm[1][:-len(dotwav)]+dotmp3
+        toMove.append(tm)
+        scld[textToSynth.lower().encode('utf-8')+'_'+langToSynth+dotwav] = 1
+        return
     generating[k]=1
     global count
-    while gradint.fileExists(newStuff+os.sep+str(count)+"_"+langToSynth+gradint.dottxt): count += 1
-    open(newStuff+os.sep+str(count)+"_"+langToSynth+gradint.dottxt,"w").write(textToSynth[len(sporadic):].encode('utf-16'))
+    while gradint.fileExists(newStuff+os.sep+str(count)+"_"+langToSynth+dottxt): count += 1
+    open(newStuff+os.sep+str(count)+"_"+langToSynth+dottxt,"w").write(textToSynth[len(sporadic):].encode('utf-16'))
     count += 1
 
 print "Checking for new ones"
-for _,s1,s2 in samples+gradint.parseSynthVocab("vocab.txt"):
+for _,s1,s2 in samples+gradint.parseSynthVocab(gradint.vocabFile):
     if type(s1)==type([]): [maybe_cache(i) for i in s1]
     else: maybe_cache(s1)
     maybe_cache(s2)
 
+if toMove: sys.stderr.write("Renaming\n")
+for tmpfile,dest in toMove:
+    oldDest = dest
+    try:
+        rename(tmpfile,gradint.synthCache+os.sep+dest)
+    except OSError: # not a valid filename
+        while gradint.fileExists(gradint.synthCache+os.sep+("__file%d" % count)+dotwav) or gradint.fileExists(gradint.synthCache+os.sep+("__file%d" % count)+dotmp3): count += 1
+        rename(tmpfile,gradint.synthCache+os.sep+("__file%d" % count)+dotwav)
+        open(gradint.synthCache+os.sep+gradint.transTbl,"ab").write("__file%d%s %s\n" % (count,dotwav,dest))
+        dest = "__file%d%s" % (count,dotwav)
+    if testMode:
+        print oldDest
+        e=gradint.SampleEvent(gradint.synthCache+os.sep+dest)
+        t=time.time() ; e.play()
+        while time.time() < t+e.length: time.sleep(1) # in case play() is asynchronous
+    
 if count: print "Now convert the files in "+newStuff+" and re-run this script.\nYou might also want to adjust the volume if appropriate, e.g. mp3gain -r -d 6 -c *.mp3"
-else: print "No extra files needed to be made."
+elif not toMove: print "No extra files needed to be made."
+else: print "All done"

samples/utils/trace.py

+#!/usr/bin/env python2
+
+# trace.py: script to generate raytraced animations of Gradint lessons
+# Version 1.32 (c) 2018-19,2021 Silas S. Brown.  License: GPL
+
+#  The Disney Pixar film "Inside Out" (2015) represented
+#  memories as spheres.  I don't have their CGI models, but
+#  we can do spheres in POV-Ray and I believe that idea is
+#  simple enough to be in the public domain (especially if
+#  NOT done like Pixar did it) - hopefully this might show
+#  some people how Gradint's method is supposed to work
+#  (especially if they've seen the Inside Out film).
+
+# This script generates the POV-Ray scenes from a lesson.
+# Gradint is run normally (passing any extra command-line arguments on,
+# must include outputFile so audio can be included in the animation)
+# and then the animation is written to /tmp/gradint.mp4.
+
+# Optionally add a static image representing each word (image will be
+# placed onto the spheres, and projected onto the back wall
+# when that word is being recalled)
+# e.g. word1_en.wav, word1_zh.wav, word1.jpg
+# (or png or gif).
+
+# Optionally add an mp4 video of a word in a particular language
+# e.g. word1_en.mp4 (probably best synchronised to word1_en.wav),
+# can also do this for commentsToAdd and orderlessCommentsToAdd files
+
+# Requires POV-Ray, ffmpeg, and the Python packages vapory
+# and futures (use sudo pip install futures vapory) -
+# futures is used to run multiple instances of POV-Ray on
+# multi-core machines.
+
+from optparse import OptionParser
+parser = OptionParser()
+parser.add_option("--fps",default=15,dest="theFPS",
+                  help="Frames per second (10 is insufficient for fast movement, so recommend at least 15)")
+parser.add_option("--res",default=480,
+                  help="Y-resolution: 240=NTSC VCD, 288=PAL VCD, 480=DVD, 607=WeChat channel, 720=Standard HD (Blu-Ray), 1080=Full HD (Blu-Ray)")
+parser.add_option("--translucent",action="store_true",default=False,dest="translucent_spheres_when_picture_visible",
+                  help="Translucent spheres when picture visible (slows down rendering but is better quality)")
+parser.add_option("--minutes",default=0,
+                  help="Maximum number of minutes to render (0 = unlimited, the default; can limit for test runs)")
+parser.add_option("--quality",default=9,dest="povray_quality",
+                  help="POVRay quality setting, default 9: 1=ambient light only, 2=lighting, 4,5=shadows, 8=reflections 9-11=radiosity etc")
+
+options, args = parser.parse_args()
+globals().update(options.__dict__)
+theFPS,res,minutes,povray_quality = int(theFPS),int(res),int(minutes),int(povray_quality)
+
+if res in [240,288]:
+    width_height_antialias = (352,res,0.3) # VCD.  antialias=None doesn't look very good at 300x200, cld try it at higher resolutions (goes to the +A param, PovRay default is 0.3 if -A specified without param; supersample (default 9 rays) if colour differs from neighbours by this amount)
+elif res==480: width_height_antialias = (640,480,0.001) # 480p (DVD)
+elif res==607: width_height_antialias = (1080,607,None) # WeChat Channels
+elif res==720: width_height_antialias = (1280,720,None) # Standard HD (Blu-Ray)
+elif res==1920: width_height_antialias = (1920,1080,None) # Full HD (Blu-Ray)
+else: raise Exception("Unknown vertical resolution specified: "+repr(res))
+debug_frame_limit = minutes * theFPS * 60
+
+import sys,os,traceback
+oldName = __name__ ; from vapory import * ; __name__ = oldName
+from concurrent.futures import ProcessPoolExecutor
+
+assert os.path.exists("gradint.py"), "You must move trace.py to the top-level Gradint directory and run it from there"
+sys.argv = [sys.argv[0]]+args
+import gradint
+assert gradint.outputFile, "You must run trace.py with gradint parameters that include outputFile"
+
+try: xrange
+except: xrange = range
+S,B = gradint.S,gradint.B
+
+class MovableParam:
+    def __init__(self): self.fixed = []
+    def fixAt(self,t,value):
+        while any(x[0]==t and not x[1]==value for x in self.fixed): t += 0.2
+        self.fixed.append((t,value))
+    def getPos(self,t):
+        assert self.fixed, "Should fixAt before getPos"
+        self.fixed.sort()
+        for i in xrange(len(self.fixed)):
+            if self.fixed[i][0] >= t:
+                if i: # interpolate
+                    if self.fixed[i-1][1]==None: return None
+                    duration = self.fixed[i][0]-self.fixed[i-1][0]
+                    progress = t-self.fixed[i-1][0]
+                    return (self.fixed[i][1]*progress + self.fixed[i-1][1]*(duration-progress))*1.0/duration
+                else: return self.fixed[i][1] # start position
+        return self.fixed[-1][1]
+
+class MovablePos:
+    def __init__(self): self.x,self.y,self.z = MovableParam(),MovableParam(),MovableParam()
+    def fixAt(self,t,*args):
+        if args[0]==None: x=y=z=None
+        else: x,y,z = args
+        self.x.fixAt(t,x),self.y.fixAt(t,y),self.z.fixAt(t,z)
+    def getPos(self,t):
+        r=(self.x.getPos(t),self.y.getPos(t),self.z.getPos(t))
+        if r==(None,None,None): return None
+        else: return r
+
+SceneObjects = set()
+
+class MovableSphere(MovablePos):
+    def __init__(self,radius=0.5,colour="prompt",imageFilename=None):
+        MovablePos.__init__(self)
+        self.colour = colour
+        self.imageFilename = imageFilename
+        self.radius = MovableParam()
+        self.radius.fixAt(-1,radius)
+        SceneObjects.add(self)
+    # fixAt(t,x,y,z) inherited
+    def obj(self,t):
+        pos = self.getPos(t)
+        if not pos: return # not in scene at this time
+        r = self.radius.getPos(t)
+        if self.imageFilename:
+            if translucent_spheres_when_picture_visible and bkgScrFade.getPos(t) < 1: transmittence = 0.5
+            else: transmittence = 0.3
+            img = wallPic(t,self.imageFilename) # if a video is playing whose key image matches ours, 'back-copy' the video frame (TODO: do this only on the correct L1 or L2 sphere?)
+            if not img: img = self.imageFilename
+            return Sphere(list(pos),r,colour(self.colour,t),Texture(Pigment(ImageMap('"'+S(img)+'"',"once","interpolate 2","transmit all "+str(transmittence)),'scale',[1.5*r,1.5*r,1],'translate',list(pos),'translate',[-.75*r,-.75*r,0])))
+        else: return Sphere(list(pos),r,colour(self.colour,t))
+
+class ObjCollection:
+    def __init__(self): self.objs = set()
+    def add(self,obj,dx,dy,dz): self.objs.add((obj,dx,dy,dz))
+    def get(self,dx,dy,dz): # should be small so:
+        for o,ddx,ddy,ddz in self.objs:
+            if (ddx,ddy,ddz) == (dx,dy,dz): return o
+    def fixAt(self,t,*args):
+        if args[0]==None: x=y=z=None
+        else: x,y,z = args
+        for obj,dx,dy,dz in self.objs:
+            if args==[None]: obj.fixAt(t,None,None,None)
+            else: obj.fixAt(t,x+dx,y+dy,z+dz)
+
+eventTrackers = {}
+def EventTracker(rowNo,imageFilename=None):
+    if not rowNo in eventTrackers:
+        eventTrackers[rowNo] = ObjCollection()
+        eventTrackers[rowNo].add(MovableSphere(1,"l1",imageFilename),-1,0,0)
+        eventTrackers[rowNo].add(MovableSphere(1,"l2",imageFilename),+1,0,0)
+        eventTrackers[rowNo].numRepeats = 0
+    return eventTrackers[rowNo]
+rCache = {}
+def repeatSphere(rowNo,numRepeats=0):
+    if not (rowNo,numRepeats) in rCache:
+        rCache[(rowNo,numRepeats)] = MovableSphere(0.1,"prompt")
+    return rCache[(rowNo,numRepeats)]
+def addRepeat(rowNo,t=0,length=0):
+    et = EventTracker(rowNo)
+    rpt = repeatSphere(rowNo,et.numRepeats)
+    if length:
+        rpt.fixAt(-1,None) # not exist yet (to save a tiny bit of POVRay computation)
+        rpt.fixAt(t-1,4*rowNo+1,0,61) # behind far wall
+        rpt.fixAt(t,4*rowNo-1,0,0) # ready to be 'batted'
+        et.fixAt(t,4*rowNo,0,10) # we're at bottom
+        camera_lookAt.fixAt(t,4*rowNo,0,10)
+        camera_lookAt.fixAt(t+length,4*rowNo,10,10)
+        camera_position.x.fixAt(t+length/2.0,4*rowNo)
+        # careful with Y : try to avoid sudden vertical motion between 2 sequences
+        camera_position.y.fixAt(t+length*.2,1)
+        camera_position.y.fixAt(t+length*.8,4)
+        camera_position.z.fixAt(t+length*.2,-10)
+        camera_position.z.fixAt(t+length*.8,-5)
+    et.add(rpt,0,1+0.2*et.numRepeats,0) # from now on we keep this marker
+    et.fixAt(t+length,4*rowNo,10,10) # at end of repeat (or at t=0) we're at top, and the repeat marker is in place
+    et.numRepeats += 1
+
+camera_position = MovablePos()
+camera_lookAt = MovablePos()
+def cam(t): return Camera('location',list(camera_position.getPos(t)),'look_at',list(camera_lookAt.getPos(t)))
+def lights(t): return [LightSource([camera_position.x.getPos(t)+10, 15, -20], [1.3, 1.3, 1.3])]
+
+background_screen = [] # (startTime,endTime,pictureName,pictureActual)
+background_screen_size = 50
+bkgScrFade = MovableParam() ; bkgScrFade.fixAt(-1,1)
+bkgScrX = MovableParam()
+def wallPic(t,ifImg=None):
+    if bkgScrFade.getPos(t) == 1: return # no picture if we're faded out
+    found = None
+    for st,et,img,pic in background_screen:
+        if st <= t: found = (st,et,img,pic)
+        elif st > t: break
+    if found:
+        st,et,img,pic = found
+        if ifImg and not img==ifImg: return
+        if B(pic).endswith(B(os.extsep+"mp4")):
+            # need to take single frame
+            T = min(t,et-1.0/theFPS)-st # don't go past last frame
+            out = B(pic)[:-4]+B("-"+str(T)+os.extsep+"jpg")
+            while T > 0 and not os.path.exists(out): # (TODO: if its frame rate is low enough, we might already have the same frame even at a slightly different T)
+                cmd = "ffmpeg -n -threads 1 -accurate_seek -ss "+str(T)+" -i "+S(pic)+" -vframes 1 -q:v 1 "+S(out)+" </dev/null >/dev/null"
+                print (cmd)
+                os.system(cmd)
+                T -= 1.0/theFPS
+            if os.path.exists(out): return out
+            else: return None
+        else: return pic
+def wall(t):
+    picToUse = wallPic(t)
+    if picToUse: return [Plane([0, 0, 1], 60, Texture(Pigment('color', [1, 1, 1])), Texture(Pigment(ImageMap('"'+S(picToUse)+'"',"once","transmit all "+str(bkgScrFade.getPos(t))),'scale',[background_screen_size,background_screen_size,1],'translate',[bkgScrX.getPos(t)-background_screen_size/2,0,0])), Finish('ambient',0.9))]
+    else: return [Plane([0, 0, 1], 60, Texture(Pigment('color', [1, 1, 1])), Finish('ambient',0.9))] # TODO: why does this look brighter than with ImageMap at transmit all 1.0 ?
+
+ground = Plane( [0, 1, 0], -1, Texture( Pigment( 'color', [1, 1, 1]), Finish( 'phong', 0.1, 'reflection',0.4, 'metallic', 0.3))) # from vapory example
+
+def colour(c,t=None):
+    c = {"l1":[.8,1,.2],"l2":[.5,.5,.9],"prompt":[1,.6,.5]}[c] # TODO: better colours
+    if translucent_spheres_when_picture_visible and not t==None and bkgScrFade.getPos(t) < 1: return Texture(Pigment('color',c,'filter',0.7))
+    else: return Texture(Pigment('color',c))
+def scene(t):
+    """ Returns the scene at time 't' (in seconds) """
+    return Scene(cam(t), lights(t) + wall(t) + [ground] + [o for o in [x.obj(t) for x in SceneObjects] if not o==None])
+
+def Event_draw(self,startTime,rowNo,inRepeat): pass
+gradint.Event.draw = Event_draw
+
+def CompositeEvent_draw(self,startTime,rowNo,inRepeat):
+  if self.eventList:
+    t = startTime
+    for i in self.eventList:
+      i.draw(t,rowNo,True)
+      t += i.length
+    if inRepeat: return
+    # Call addRepeat, but postpone the start until the
+    # first loggable event, to reduce rapid camera mvt
+    st0 = startTime
+    for i in self.eventList:
+      if i.makesSenseToLog(): break
+      else: startTime += i.length
+    if startTime==t: startTime = st0 # shouldn't happen
+    addRepeat(rowNo,startTime,t-startTime)
+gradint.CompositeEvent.draw=CompositeEvent_draw
+
+def Event_colour(self,language):
+    if self.makesSenseToLog():
+      if language==gradint.firstLanguage: return "l1"
+      else: return "l2"
+    else: return "prompt"
+gradint.Event.colour = Event_colour
+
+def eDraw(startTime,length,rowNo,colour):
+    minR = 0.5
+    if colour in ["l1","l2"]:
+        if colour=="l1": delta = -1
+        else: delta = +1
+        et = EventTracker(rowNo).get(delta,0,0)
+        r = et.radius
+        if hasattr(et,"imageFilename"):
+            background_screen.append((startTime,startTime+length,et.imageFilename,et.imageFilename))
+            bkgScrX.fixAt(startTime,4*rowNo)
+            bkgScrX.fixAt(startTime+length,4*rowNo)
+    else:
+        r = repeatSphere(rowNo,EventTracker(rowNo).numRepeats).radius
+        minR = 0.1
+    maxR = min(max(length,minR*1.5),minR*3) # TODO: vary with event's volume, so cn see the syllables? (partials can do that anyway)
+    r.fixAt(startTime,minR)
+    r.fixAt(startTime+length,minR)
+    if length/2.0 > 0.5:
+        r.fixAt(startTime+0.5,maxR)
+        # TODO: wobble in the middle?
+        r.fixAt(startTime+length-0.5,maxR)
+    else: r.fixAt(startTime+length/2.0,maxR)
+
+def SampleEvent_draw(self,startTime,rowNo,inRepeat):
+    if B(self.file).startswith(B(gradint.partialsDirectory)): l=B(self.file).split(B(os.sep))[1]
+    else: l = gradint.languageof(self.file)
+    eDraw(startTime,self.length,rowNo,self.colour(S(l)))
+gradint.SampleEvent.draw = SampleEvent_draw
+def SynthEvent_draw(self,startTime,rowNo,inRepeat): eDraw(startTime,self.length,rowNo,self.colour(self.language))
+gradint.SynthEvent.draw = SynthEvent_draw
+
+def chkImg(i):
+    if not "_" in S(i.file): return
+    for imgExt in ["gif","png","jpeg","jpg"]:
+        imageFilename = B(i.file)[:B(i.file).rindex(B("_"))]+B(os.extsep+imgExt) # TODO: we're assuming no _en etc in the image filename (projected onto both L1 and L2)
+        if os.path.exists(imageFilename):
+            return os.path.abspath(imageFilename)
+
+def runGradint():
+  gradint.gluedListTracker=[]
+  gradint.waitBeforeStart=0
+  gradint.main()
+  gradint.gluedListTracker.sort(key=lambda e:e[0].glue.length+e[0].glue.adjustment)
+  duration = 0
+  for l,row in zip(gradint.gluedListTracker,xrange(len(gradint.gluedListTracker))):
+    def check_for_pictures():
+     for gluedEvent in l:
+      event = gluedEvent.event
+      try: el=event.eventList
+      except: el=[event]
+      for j in el:
+       try: el2=j.eventList
+       except: el2=[j]
+       for i in el2:
+        if hasattr(i,"file") and B("_") in B(i.file):
+            imageFilename = chkImg(i)
+            if imageFilename:
+                return EventTracker(row,imageFilename)
+    check_for_pictures()
+    if hasattr(l[0],"timesDone"): timesDone = l[0].timesDone
+    else: timesDone = 0
+    for i in xrange(timesDone): addRepeat(row)
+    glueStart = 0
+    for i in l:
+      i.event.draw(i.getEventStart(glueStart),row,False)
+      glueStart = i.getAdjustedEnd(glueStart)
+      duration = max(duration,glueStart)
+  for t,e in gradint.lastLessonMade.events: # check for videos
+      if hasattr(e,"file") and hasattr(e,"exactLen"):
+          video = B(e.file)[:B(e.file).rindex(B(os.extsep))]+B(os.extsep+"mp4")
+          if os.path.exists(video): # overwrite static image while playing
+              i,v = chkImg(e),os.path.abspath(video)
+              if not i: i=v
+              background_screen.append((t,t+e.exactLen,i,v))
+  background_screen.sort()
+  i = 0 # more items might be inserted, so don't use range here
+  while i < len(background_screen)-1:
+      if background_screen[i][1] > background_screen[i+1][1]: # overlap: we end after next one ends: insert a jump-back-to-us after
+          background_screen.insert(i+2,(background_screen[i+1][1],background_screen[i][1],background_screen[i][2],background_screen[i][3])) # restore old after new one ends
+      if background_screen[i][1] > background_screen[i+1][0] and background_screen[i][0] < background_screen[i+1][0]: # overlap: we end after next one starts, but we start before it starts
+          background_screen[i] = (background_screen[i][0],background_screen[i+1][0],background_screen[i][2],background_screen[i][3]) # new one takes precedence
+      if background_screen[i][0]==background_screen[i+1][0]: # equal start, but next one might be longer
+          background_screen[i+1]=(background_screen[i][1],background_screen[i+1][1],background_screen[i+1][2],background_screen[i+1][3])
+      if background_screen[i][2]==background_screen[i+1][2] and background_screen[i][1]+5>=background_screen[i+1][0] and background_screen[i][1] < background_screen[i+1][0]:
+          # avoid turning off for 5 seconds or less if showing the same image (or a video of it)
+          background_screen.insert(i+1,(background_screen[i][1],background_screen[i+1][0],background_screen[i][2],background_screen[i][2])) # just the image
+      i += 1
+  for i in xrange(len(background_screen)):
+      startTime,endTime,picName,img = background_screen[i]
+      if i and startTime > background_screen[i-1][1] + 0.5:
+          bkgScrFade.fixAt(startTime,1) # start faded out
+      # else (less than 0.5sec between images) don't try to start faded out
+      fadeOutTime = endTime
+      if i<len(background_screen)-1:
+          if endTime + 0.5 > background_screen[i+1][0]:
+              fadeOutTime = None # as above (< 0.5sec between images)
+          else: fadeOutTime = max(fadeOutTime,min(background_screen[i+1][0]-1,fadeOutTime+5))
+          if not fadeOutTime == None:
+              # don't move the screen during any extended fade-out:
+              for ii in xrange(len(bkgScrX.fixed)):
+                  if bkgScrX.fixed[ii][0]==endTime:
+                      bkgScrX.fixed[ii]=((fadeOutTime,bkgScrX.fixed[ii][1]))
+                      break
+      if not fadeOutTime==None: bkgScrFade.fixAt(fadeOutTime,1)
+      if endTime >= startTime+0.5:
+          bkgScrFade.fixAt(startTime+0.5,0.3)
+          bkgScrFade.fixAt(endTime-0.5,0.3)
+      else:
+          bkgScrFade.fixAt((startTime+endTime)/2.0,0.3)
+  return duration
+
+def tryFrame(f):
+    frame,numFrames = f
+    print ("Making frame "+str(frame)+" of "+str(numFrames))
+    try:
+        try: os.mkdir("/tmp/"+repr(frame)) # vapory writes a temp .pov file and does not change its name per process, so better be in a process-unique directory
+        except: pass
+        os.chdir("/tmp/"+repr(frame))
+        scene(frame*1.0/theFPS).render(width=width_height_antialias[0], height=width_height_antialias[1], antialiasing=width_height_antialias[2], quality=povray_quality, outfile="/tmp/frame%05d.png" % frame)
+        # TODO: TURN OFF JITTER with -J if using anti-aliasing in animations
+        os.chdir("/tmp") ; os.system('rm -r '+repr(frame))
+        return None
+    except:
+        if frame==0: raise
+        traceback.print_exc()
+        sys.stderr.write("Frame %d render error, will skip\n" % frame)
+        return "cp /tmp/frame%05d.png /tmp/frame%05d.png" % (frame-1,frame)
+
+def main():
+    executor = ProcessPoolExecutor()
+    duration = runGradint()
+    numFrames = int(duration*theFPS)
+    if debug_frame_limit: numFrames=min(numFrames,debug_frame_limit)
+    # TODO: pickle all MovableParams so can do the rendering on a different machine than the one that makes the Gradint lesson?
+    for c in list(executor.map(tryFrame,[(frame,numFrames) for frame in xrange(numFrames)]))+[
+        "ffmpeg -nostdin -y -framerate "+repr(theFPS)+" -i /tmp/frame%05d.png -i "+gradint.outputFile+" -movflags faststart -pix_fmt yuv420p -filter_complex tpad=stop=-1:stop_mode=clone -shortest /tmp/gradint.mp4 && if [ -d /Volumes ]; then open /tmp/gradint.mp4; fi" #  (could alternatively run with -vcodec huffyuv /tmp/gradint.avi for lossless, insead of --movflags etc, but will get over 6 gig and may get A/V desync problems in mplayer/VLC that -delay doesn't fix, however -b:v 1000k seems to look OK; for WeChat etc you need to recode to h.264, and for HTML 5 video need recode to WebM (but ffmpeg -c:v libvpx no good if not compiled with support for those libraries; may hv to convert on another machine i.e. ffmpeg -i gradint.mp4 -vf scale=320:240 -c:v libvpx -b:v 500k gradint.webm))
+        ]:
+        if c: # patch up skipped frames, then run ffmpeg
+            print (c) ; os.system(c)
+    for f in xrange(numFrames): os.remove("/tmp/frame%05d.png" % f) # wildcard from command line could get 'argument list too long' on BSD etc
+if __name__=="__main__": main()
+else: print (__name__)

gradint-build/samples/utils/transliterate.py → samples/utils/transliterate.py

+#!/usr/bin/env python2
+
 # transliterate.py - print a 2nd-language-transliterated version of vocab.txt and any .txt pairs in samples
 # (may be useful for grepping, loading to Latin-only PDA, etc)
 # (note: leaves comments untransliterated, + may not translit all text if gradint is set up so a transliterating synth will not be used)

server/README.txt

+Gradint server tools
+--------------------
+
+gradint.cgi - CGI script to run Gradint completely online
+(but just synthesized vocabulary for now).
+Not integrated with email-lesson.sh below, because that needs a
+large vocab (which probably needs non web-based admin)
+
+email-lesson* - scripts that can help you to
+automatically distribute daily lessons to students
+using a web server with reminder emails
+
+vocab2html.py - make an HTML index for a synth cache,
+with the help of a vocab.txt (you can also use it with espeak.cgi)
+
+samples.cgi - CGI script to browse a samples directory
+(make sure you have permission to publish your recordings,
+or that the site is not publically viewable)
+
+espeak.cgi - script that lets a Web user play with espeak options
+
+Other files - see description at the top of the file

server/cantonese.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#  (should work with either Python 2 or Python 3)
+
+# cantonese.py - Python functions for processing Cantonese transliterations
+# (uses eSpeak and Gradint for help with some of them)
+
+# v1.48 (c) 2013-15,2017-24 Silas S. Brown.  License: GPL
+
+cache = {} # to avoid repeated eSpeak runs,
+# zi -> jyutping or (pinyin,) -> translit
+dryrun_mode = False # True = prepare to populate cache in batch
+jyutping_dryrun,pinyin_dryrun = set(),set()
+
+import re, pickle, os, sys
+if '--cache' in sys.argv:
+  cache_fname = sys.argv[sys.argv.index('--cache')+1]
+else: cache_fname = os.environ.get("JYUTPING_CACHE","/tmp/.jyutping-cache")
+try: cache = pickle.Unpickler(open(cache_fname,"rb")).load()
+except: pass
+
+extra_zhy_dict = { # TODO: add these to the real zhy_list in eSpeak
+  u"\u9c85":"bat3",u"\u9b81":"bat3",
+}
+
+def S(v): # make sure it's a string in both Python 2 and 3
+  if type("")==type(u""): # Python 3
+    try: return v.decode('utf-8') # in case it's bytes
+    except: return v
+  else: return v
+def B(v): # make sure it's bytes in Python 3, str in Python 2
+  if type(v)==type(u""): return v.encode('utf-8')
+  return v
+
+def get_jyutping(hanzi,mustWork=1):
+  if not type(hanzi)==type(u""): hanzi=hanzi.decode('utf-8')
+  for k,v in extra_zhy_dict.items(): hanzi=hanzi.replace(k,v)
+  global espeak
+  if not espeak:
+      espeak = import_gradint().ESpeakSynth()
+      if not espeak.works_on_this_platform(): # must call
+          raise Exception("espeak.works_on_this_platform")
+      assert espeak.supports_language("zhy")
+
+  global jyutping_dryrun
+  if dryrun_mode:
+      if not hanzi in cache: jyutping_dryrun.add(hanzi)
+      return "aai1" # placeholder value
+  elif jyutping_dryrun:
+      jyutping_dryrun = list(jyutping_dryrun)
+      vals = espeak.transliterate_multiple("zhy",jyutping_dryrun,0)
+      assert len(jyutping_dryrun)==len(vals)
+      for k,v in zip(jyutping_dryrun,vals):
+        cache[k]=S(v).replace("7","1").lower() # see below
+      jyutping_dryrun = set()
+  if hanzi in cache: jyutping = cache[hanzi]
+  else: cache[hanzi] = jyutping = S(espeak.transliterate("zhy",hanzi,forPartials=0)).replace("7","1").lower() # .lower() needed because espeak sometimes randomly capitalises e.g. 2nd hanzi of 'hypocrite' (Mandarin xuwei de ren)
+  if mustWork: assert jyutping.strip(), "No translit. result for "+repr(hanzi)
+  elif not jyutping.strip(): jyutping=""
+  return jyutping
+espeak = 0
+
+def hanzi_only(unitext): return u"".join(filter(lambda x:0x4e00<=ord(x)<0xa700 or ord(x)>=0x10000, list(unitext)))
+def py2nums(pinyin):
+  if not type(pinyin)==type(u""):
+    pinyin = pinyin.decode('utf-8')
+  if not pinyin.strip(): return ""
+  global pinyin_dryrun
+  if pinyin_dryrun:
+    pinyin_dryrun = list(pinyin_dryrun)
+    vals = espeak.transliterate_multiple("zh",pinyin_dryrun,0)
+    assert len(pinyin_dryrun)==len(vals)
+    for i in range(len(pinyin_dryrun)):
+      cache[(pinyin_dryrun[i],)]=vals[i]
+    pinyin_dryrun = set()
+  if (pinyin,) in cache: pyNums = cache[(pinyin,)]
+  else: pyNums = espeak.transliterate("zh",pinyin,forPartials=0) # (this transliterate just does tone marks to numbers, adds 5, etc; forPartials=0 because we DON'T want to change letters like X into syllables, as that won't happen in jyutping and we're going through it tone-by-tone)
+  assert pyNums and pyNums.strip(), "espeak.transliterate returned %s for %s" % (repr(pyNums),repr(pinyin))
+  return re.sub("a$","a5",re.sub("(?<=[a-zA-Z])er([1-5])",r"e\1r5",S(pyNums)))
+if type(u"")==type(""): # Python 3
+  getNext = lambda gen: gen.__next__()
+else: getNext = lambda gen: gen.next()
+def adjust_jyutping_for_pinyin(hanzi,jyutping,pinyin):
+  # If we have good quality (proof-read etc) Mandarin pinyin, this can sometimes improve the automatic Cantonese transcription
+  if not type(hanzi)==type(u""): hanzi = hanzi.decode('utf-8')
+  hanzi = hanzi_only(hanzi)
+  if not re.search(py2j_chars,hanzi): return jyutping
+  pinyin = re.findall('[A-Za-z]*[1-5]',py2nums(pinyin))
+  if not len(pinyin)==len(hanzi): return jyutping # can't fix
+  jyutping = S(jyutping)
+  i = 0 ; tones = re.finditer('[1-7]',jyutping) ; j2 = []
+  for h,p in zip(list(hanzi),pinyin):
+    try: j = getNext(tones).end()
+    except StopIteration: return jyutping # one of the hanzi has no Cantonese reading in our data: we'll warn "failed to fix" below
+    j2.append(jyutping[i:j]) ; i = j
+    if h in py2j and p.lower() in py2j[h]: j2[-1]=j2[-1][:re.search("[A-Za-z]*[1-7]$",j2[-1]).start()]+py2j[h][p.lower()]
+  return "".join(j2)+jyutping[i:]
+py2j={
+u"\u4E2D":{"zhong1":"zung1","zhong4":"zung3"},
+u"\u4E3A\u70BA":{"wei2":"wai4","wei4":"wai6"},
+u"\u4E50\u6A02":{"le4":"lok6","yue4":"ngok6"},
+u"\u4EB2\u89AA":{"qin1":"can1","qing4":"can3"},
+u"\u4EC0":{"shen2":"sam6","shi2":"sap6"}, # unless zaap6
+u"\u4F20\u50B3":{"chuan2":"cyun4","zhuan4":"zyun6"},
+u"\u4FBF":{"bian4":"bin6","pian2":"pin4"},
+u"\u5047":{"jia3":"gaa2","jia4":"gaa3"},
+u"\u5174\u8208":{"xing1":"hing1","xing4":"hing3"},
+# u"\u5207":{"qie4":"cai3","qie1":"cit3"}, # WRONG (rm'd v1.17).  It's cit3 in re4qie4.  It just wasn't in yiqie4 (which zhy_list has as an exception anyway)
+u"\u521B\u5275":{"chuang1":"cong1","chuang4":"cong3"},
+u"\u53EA":{"zhi1":"zek3","zhi3":"zi2"},
+u"\u53F7\u865F":{"hao4":"hou6","hao2":"hou4"},
+u"\u548C":{"he2":"wo4","he4":"wo6"},
+u"\u54BD":{"yan1":"jin1","yan4":"jin3","ye4":"jit3"},
+u"\u5708":{"juan4":"gyun6","quan1":"hyun1"},
+u"\u597D":{"hao3":"hou2","hao4":"hou3"},
+u"\u5C06\u5C07":{"jiang1":"zoeng1","jiang4":"zoeng3"},
+u"\u5C11":{"shao3":"siu2","shao4":"siu3"},
+u"\u5DEE":{"cha4":"caa1","cha1":"caa1","chai1":"caai1"},
+u"\u5F37\u5F3A":{"qiang2":"koeng4","qiang3":"koeng5"},
+u"\u62C5\u64D4":{"dan1":"daam1","dan4":"daam3"},
+u"\u6323\u6399":{"zheng4":"zaang6","zheng1":"zang1"},
+u"\u6570\u6578":{"shu3":"sou2","shu4":"sou3"},
+u"\u671D":{"chao2":"ciu4","zhao1":"ziu1"},
+u"\u6ED1":{"hua2":"waat6","gu3":"gwat1"},
+u"\u6F02":{"piao1":"piu1","piao3 piao4":"piu3"},
+u"\u76DB":{"sheng4":"sing6","cheng2":"sing4"},
+u"\u76F8":{"xiang1":"soeng1","xiang4":"soeng3"},
+u"\u770B":{"kan4":"hon3","kan1":"hon1"},
+u"\u79CD\u7A2E":{"zhong3":"zung2","zhong4":"zung3"},
+u"\u7EF7\u7E43":{"beng1":"bang1","beng3":"maang1"},
+u"\u8208":{"xing1":"hing1","xing4":"hing3"},
+u"\u843D":{"luo1 luo4 lao4":"lok6","la4":"laai6"},
+u"\u8457":{"zhu4":"zyu3","zhuo2":"zoek3","zhuo2 zhao2 zhao1 zhe5":"zoek6"},
+u"\u8981":{"yao4":"jiu3","yao1":"jiu1"},
+u"\u89C1\u898B":{"jian4":"gin3","xian4":"jin6"},
+u"\u89C9\u89BA":{"jue2":"gok3","jiao4":"gaau3"},
+u"\u8B58\u8BC6":{"shi2 shi4":"sik1","zhi4":"zi3"},
+u"\u8ABF\u8C03":{"diao4":"diu6","tiao2":"tiu4"},
+u"\u91CF":{"liang2":"loeng4","liang4":"loeng6"},
+u"\u9577\u957F":{"chang2":"coeng4","zhang3":"zoeng2"},
+u"\u9593\u95F4":{"jian1":"gaan1","jian4":"gaan3"},
+u"\u96BE\u96E3":{"nan2":"naan4","nan4":"naan6"}}
+for k in list(py2j.keys()):
+   if len(k)>1:
+    for c in list(k): py2j[c]=py2j[k]
+    del py2j[k]
+for _,v in py2j.items():
+  for k in list(v.keys()):
+    if len(k.split())>1:
+      for w in k.split(): v[w]=v[k]
+      del v[k]
+py2j_chars = re.compile(u'['+''.join(list(py2j.keys()))+']')
+
+def jyutping_to_lau(j):
+  j = S(j).lower().replace("j","y").replace("z","j")
+  for k,v in jlRep: j=j.replace(k,v)
+  return j.lower().replace("ohek","euk")
+def jyutping_to_lau_java(jyutpingNo=2,lauNo=1):
+  # for annogen.py 3.29+ --annotation-postprocess to ship Jyutping and generate Lau at runtime
+  return 'if(annotNo=='+str(jyutpingNo)+'||annotNo=='+str(lauNo)+'){m=Pattern.compile("<rt>(.*?)</rt>").matcher(r);sb=new StringBuffer();while(m.find()){String r2=(annotNo=='+str(jyutpingNo)+'?m.group(1).replaceAll("([1-7])(.)","$1&shy;$2"):(m.group(1)+" ").toLowerCase().replace("j","y").replace("z","j")'+''.join('.replace("'+k+'","'+v+'")' for k,v in jlRep)+'.toLowerCase().replace("ohek","euk").replaceAll("([1-7])","<sup>$1</sup>-").replace("- "," ").replaceAll(" $","")),tmp=m.group(1).substring(0,1);if(annotNo=='+str(lauNo)+'&&tmp.equals(tmp.toUpperCase()))r2=r2.substring(0,1).toUpperCase()+r2.substring(1);m.appendReplacement(sb,"<rt>"+r2+"</rt>");}m.appendTail(sb); r=sb.toString();}' # TODO: can probably go faster with mapping for some of this
+def incomplete_lau_to_jyutping(l):
+  # incomplete: assumes Lau didn't do the "aa" -> "a" rule
+  l = S(l).lower().replace("euk","ohek")
+  for k,v in ljRep: l=l.replace(k,v)
+  return l.lower().replace("j","z").replace("y","j")
+def incomplete_lau_to_yale_u8(l): return jyutping_to_yale_u8(incomplete_lau_to_jyutping(l))
+jlRep = [(unchanged,unchanged.upper()) for unchanged in "aai aau aam aang aan aap aat aak ai au am ang an ap at ak a ei eng ek e iu im ing in ip it ik i oi ong on ot ok ung uk".split()] + [("eoi","UI"),("eon","UN"),("eot","UT"),("eok","EUK"),("oeng","EUNG"),("oe","EUH"),("c","ch"),("ou","O"),("o","OH"),("yu","UE"),("u","OO")]
+jlRep.sort(key=lambda a:-len(a[0])) # longest 1st
+# u to oo includes ui to ooi, un to oon, ut to oot
+# yu to ue includes yun to uen and yut to uet
+# drawing from the table on http://www.omniglot.com/writing/cantonese.htm plus this private communication:
+# Jyutping "-oeng" maps to Sidney Lau "-eung".
+# Jyutping "jyu" maps to Sidney Lau "yue". (consequence of yu->ue, j->y)
+ljRep=[(b.lower(),a.upper()) for a,b in jlRep]
+ljRep.sort(key=lambda a:-len(a[0])) # longest 1st
+
+def ping_or_lau_to_syllable_list(j): return re.sub(r"([1-9])(?![0-9])",r"\1 ",re.sub(r"[!-/:-@^-`]"," ",S(j))).split()
+
+def hyphenate_ping_or_lau_syl_list(sList,groupLens=None):
+    if type(sList) in [str,type(u"")]:
+        sList = ping_or_lau_to_syllable_list(sList)
+    return hyphenate_syl_list(sList,groupLens)
+def hyphenate_yale_syl_list(sList,groupLens=None):
+    # (if sList is a string, the syllables must be space-separated,
+    #  which will be the case if to_yale functions below are used)
+    if not type(sList)==list: sList = sList.split()
+    return hyphenate_syl_list(sList,groupLens)
+def hyphenate_syl_list(sList,groupLens=None):
+    assert type(sList) == list
+    if '--hyphenate-all' in sys.argv: groupLens = [len(sList)]
+    elif not groupLens: groupLens = [1]*len(sList) # don't hyphenate at all if we don't know
+    else: assert sum(groupLens) == len(sList), "sum("+repr(groupLens)+")!=len("+repr(sList)+")"
+    r = [] ; start = 0
+    for g in groupLens:
+        r.append("-".join(S(x) for x in sList[start:start+g]))
+        start += g
+    return " ".join(r)
+    
+def jyutping_to_yale_TeX(j): # returns space-separated syllables
+  ret=[]
+  for syl in ping_or_lau_to_syllable_list(S(j).lower().replace("eo","eu").replace("oe","eu").replace("j","y").replace("yyu","yu").replace("z","j").replace("c","ch")):
+    vowel=lastVowel=None
+    for i in range(len(syl)):
+      if syl[i] in "aeiou":
+        vowel=i ; break
+    if vowel==None and re.match(r"h?(m|ng)[456]",syl): # standalone nasal syllables
+      vowel = syl.find('m')
+      if vowel<0: vowel = syl.index('n')
+      lastVowel = syl.find('g')
+      if lastVowel<0: lastVowel = vowel
+    if vowel==None:
+      ret.append(syl.upper()) ; continue # English word or letter in the Chinese?
+    if syl[vowel:vowel+2] == "aa" and (len(syl)<vowel+2 or syl[vowel+2] in "123456"):
+      syl=syl[:vowel]+syl[vowel+1:] # final aa -> a
+    # the tonal 'h' goes after all the vowels but before any consonants:
+    for i in range(len(syl)-1,-1,-1):
+      if syl[i] in "aeiou":
+        lastVowel=i ; break
+    if syl[-1] in "1234567":
+      # get_jyutping replaces 7 with 1 because zhy_list is
+      # more Canton-type than Hong Kong-type Cantonese and
+      # there is considerable disagreement on which "1"s
+      # should be "7"s, but if you pass any "7" into the
+      # jyutping_to_yale functions we can at least process
+      # it here:
+      tone = ["\=",r"\'","",r"\`",r"\'","",r"\`"][int(syl[-1])-1]
+      if syl[-1] in "456":
+        syl=syl[:lastVowel+1]+"h"+syl[lastVowel+1:]
+      ret.append((syl[:vowel]+tone+syl[vowel:-1]).replace(r"\=i",r"\=\i{}").replace(r"\=I",r"\=\I{}"))
+    else: ret.append(syl.upper()) # English word or letter in the Chinese?
+  return ' '.join(ret)
+
+def jyutping_to_yale_u8(j): # returns space-separated syllables
+  import unicodedata
+  def mysub(z,l):
+    for x,y in l:
+      z = re.sub(re.escape(x)+r"(.)",r"\1"+y,z)
+    return z
+  if type(u"")==type(""): U=str # Python 3
+  else: # Python 2
+    def U(x):
+      try: return x.decode('utf-8') # might be an emoji pass-through
+      except: return x # already Unicode
+  return unicodedata.normalize('NFC',mysub(U(jyutping_to_yale_TeX(j).replace(r"\i{}","i").replace(r"\I{}","I")),[(r"\`",u"\u0300"),(r"\'",u"\u0301"),(r"\=",u"\u0304")])).encode('utf-8')
+
+def superscript_digits_TeX(j):
+  # for jyutping and Sidney Lau
+  j = S(j)
+  for digit in "123456789": j=j.replace(digit,r"\raisebox{-0.3ex}{$^"+digit+r"$}\hspace{0pt}")
+  return j
+
+def superscript_digits_HTML(j):
+  j = S(j)
+  for digit in "123456789": j=j.replace(digit,"<sup>"+digit+"</sup>")
+  return j
+
+def superscript_digits_UTF8(j):
+  # WARNING: not all fonts have all digits; many have only the first 3.  superscript_digits_HTML might be better for browsers, even though it does produce more bytes.
+  j = S(j)
+  for digit in range(1,10): j=j.replace(str(digit),S(u"¹²³⁴⁵⁶⁷⁸⁹"[digit-1].encode('utf-8')))
+  if type(j)==type(u""): j=j.encode('utf-8') # Python 3
+  return j
+
+def import_gradint():
+    global gradint
+    try: return gradint
+    except: pass
+    # when importing gradint, make sure no command line
+    tmp,sys.argv = sys.argv,sys.argv[:1]
+    import gradint
+    sys.argv = tmp
+    gradint.espeak_preprocessors = {}
+    return gradint
+
+def do_song_subst(hanzi_u8): return B(hanzi_u8).replace(unichr(0x4f7f).encode('utf-8'),unichr(0x38c8).encode('utf-8')) # Mandarin shi3 (normally jyutping sai2) is usually si3 in songs, so substitute a rarer character that unambiguously has that reading before sending to get_jyutping
+
+if __name__ == "__main__":
+    # command-line use: output Lau for each line of stdin
+    # (or Yale if there's a --yale in sys.argv, or both
+    # with '#' separators if --yale#lau in sys.argv,
+    # also --yale#ping and --yale#lau#ping accepted);
+    # if there's a # in the line, assume it's hanzi#pinyin
+    # (for annogen.py --reannotator="##python cantonese.py")
+    lines = sys.stdin.read().replace("\r\n","\n").split("\n")
+    if lines and not lines[-1]: del lines[-1]
+    dryrun_mode = True
+    def songSubst(l):
+      if '--song-lyrics' in sys.argv: l=do_song_subst(l)
+      return l
+    for l in lines:
+      if '#' in l: l,pinyin = l.split('#')
+      else: pinyin = None
+      get_jyutping(songSubst(l))
+      if pinyin and not type(pinyin)==type(u""):
+        pinyin = pinyin.decode('utf-8')
+      if pinyin and not (pinyin,) in cache:
+        pinyin_dryrun.add(pinyin)
+        for w in pinyin.split():
+          for h in w.split('-'):
+            pinyin_dryrun.add(h)
+    dryrun_mode = False
+    for l in lines:
+      if '#' in l: l,pinyin = l.split('#')
+      else: pinyin = None
+      jyutping = get_jyutping(songSubst(l),0)
+      if not jyutping: groupLens = None # likely a Unihan-only 'fallback readings' zi that has no Cantonese
+      elif pinyin:
+        jyutping = adjust_jyutping_for_pinyin(l,jyutping,pinyin)
+        groupLens = [0]
+        for syl,space in re.findall('([A-Za-z]*[1-5])( *)',' '.join('-'.join(py2nums(h) for h in w.split('-')) for w in pinyin.split())): # doing it this way so we're not relying on espeak transliterate_multiple to preserve spacing and hyphenation
+          groupLens[-1] += 1
+          if space: groupLens.append(0)
+        if not groupLens[-1]: groupLens=groupLens[:-1]
+        lenWanted = len(ping_or_lau_to_syllable_list(jyutping))
+        if sum(groupLens) > lenWanted: # probably silent -r to drop
+          for i,word in enumerate(py2nums(pinyin).split()):
+            if re.search("[1-5]r5",word):
+              groupLens[i] -= 1
+              if sum(groupLens)==lenWanted: break
+        if not sum(groupLens)==lenWanted:
+          sys.stderr.write("WARNING: failed to fix "+pinyin+" ("+py2nums(pinyin)+") to "+jyutping+" ("+repr(ping_or_lau_to_syllable_list(jyutping))+") from "+l+", omitting\n")
+          groupLens = None ; jyutping = ""
+      else: groupLens = None
+      if "--yale#lau" in sys.argv: print (hyphenate_yale_syl_list(jyutping_to_yale_u8(jyutping),groupLens)+"#"+superscript_digits_HTML(hyphenate_ping_or_lau_syl_list(jyutping_to_lau(jyutping),groupLens)))
+      elif '--yale#ping' in sys.argv: print (hyphenate_yale_syl_list(jyutping_to_yale_u8(jyutping),groupLens)+"#"+jyutping.replace(' ',''))
+      elif "--yale#lau#ping" in sys.argv: print (hyphenate_yale_syl_list(jyutping_to_yale_u8(jyutping),groupLens)+"#"+superscript_digits_HTML(hyphenate_ping_or_lau_syl_list(jyutping_to_lau(jyutping),groupLens))+"#"+jyutping.replace(' ',''))
+      elif "--yale" in sys.argv: print (hyphenate_yale_syl_list(jyutping_to_yale_u8(jyutping),groupLens))
+      else: print (superscript_digits_HTML(hyphenate_ping_or_lau_syl_list(jyutping_to_lau(jyutping),groupLens)))
+    try: pickle.Pickler(open(cache_fname,"wb"),-1).dump(cache)
+    except: pass

gradint-build/samples/utils/email-lesson-archive.sh → server/email-lesson-archive.sh

 #!/bin/bash
 
 # email-lesson-archive.sh - archive an old email-lesson user
-# (C) 2008 Silas S. Brown, License: GPL
+# (C) 2008,2021-22 Silas S. Brown, License: GPL
 
 if ! pwd|grep email_lesson_users >/dev/null; then
   echo "This script should be run from an email_lesson_users directory (see email-lesson.sh)"
@@ -13,29 +13,29 @@ if test "a$1" == a; then
 fi
 . config
 while ! test "a$1" == a; do
-  if test -e "$1"; then
+  if [ -e "$1" ]; then
     unset U; unset Links
     if echo "$1"|grep "^user.0*" >/dev/null; then
       # specifying by user.0* id
       export U=$1
-      export Links=$(find . -maxdepth 1 -lname $U)
-    elif ls -l --color=none "$1"|grep ' -> ' >/dev/null; then
+      Links=$(find . -maxdepth 1 -lname "$U")
+    elif find "$1" -maxdepth 0 -type l|grep . >/dev/null; then
       # specifying by symlink
-      export Links=$1
-      export U=$(ls -l --color=none "$1"|sed -e 's/.* -> //')
+      Links=$1
+      U=$(ls -l --color=none "$1"|sed -e 's/.* -> //')
     else echo "Warning: can't make sense of username $1"; fi
-    if ! test "a$U" == a; then
-      if test -e $U/lastdate; then
+    if ! [ "a$U" == a ]; then
+      if [ -e "$U/lastdate" ]; then
         if test "a$Links" == a; then export Shortname=$U; else export Shortname=$Links; fi
-        if echo $PUBLIC_HTML | grep : >/dev/null; then
-        ssh $PUBLIC_HTML_EXTRA_SSH_OPTIONS $(echo $PUBLIC_HTML|sed -e 's/:.*//') rm -v $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$(cat $U/lastdate).*
-        else rm -v $PUBLIC_HTML/$U-$(cat $U/lastdate).*
+        if echo "$PUBLIC_HTML" | grep : >/dev/null; then
+        ssh $PUBLIC_HTML_EXTRA_SSH_OPTIONS "$(echo "$PUBLIC_HTML"|sed -e 's/:.*//')" rm -v "$(echo "$PUBLIC_HTML"|sed -e 's/[^:]*://')/$U-$(cat $U/lastdate).*"
+        else rm -v "$PUBLIC_HTML/$U-$(cat "$U/lastdate")".*
         fi
       fi
-      tar -jcvf $Shortname.tbz $U $Links
+      tar -jcvf "$Shortname.tbz" "$U" $Links
       mkdir -p old
-      mv -v --backup=numbered $Shortname.tbz old/
-      rm -rf $U $Links
+      mv -v --backup=numbered "$Shortname.tbz" old/
+      rm -rf "$U" $Links
     fi
   else echo "Warning: User $1 does not exist"; fi
 shift; done

gradint-build/samples/utils/email-lesson-report.sh → server/email-lesson-report.sh

@@ -6,7 +6,7 @@
 # - report is written to standard output so you can include
 # it in a script that makes some larger HTML page
 
-# v1.1 (C) 2007, 2009 Silas S. Brown, License: GPL
+# v1.12 (C) 2007, 2009. 2021-22 Silas S. Brown, License: GPL
 
 if ! pwd|grep email_lesson_users >/dev/null; then
   echo "This script should be run from an email_lesson_users directory (see email-lesson.sh)"
@@ -14,13 +14,13 @@ if ! pwd|grep email_lesson_users >/dev/null; then
 fi
 echo '<TABLE>'
 touch -d 0:00 /dev/shm/.midnight 2>/dev/null || touch -d 0:00 /tmp/.midnight
-if test -e /dev/shm/.midnight; then export Midnight=/dev/shm/.midnight; else export Midnight=/tmp/.midnight; fi
-for P in $(ls --color=none -t */progress.txt */podcasts-to-send 2>/dev/null); do
-  if test $P -nt $Midnight; then export Em="*";else unset Em; fi
-  if echo $P | grep podcasts-to-send$ >/dev/null; then
-    zgrep -H -m 1 . $P|grep -v ^user\.|sed -e 's,/.*:,</TD><TD COLSPAN=4>,' -e "s/^/<TR><TD>$Em/" -e "s,$,</TD></TR>,"
+if [ -e /dev/shm/.midnight ]; then Midnight=/dev/shm/.midnight; else Midnight=/tmp/.midnight; fi
+for P in $(ls --color=none -t -- */progress.txt */podcasts-to-send 2>/dev/null); do
+  if test "$P" -nt $Midnight; then Em="*";else unset Em; fi
+  if echo "$P" | grep podcasts-to-send$ >/dev/null; then
+    zgrep -H -m 1 . "$P"|grep -v ^user\.|sed -e 's,/.*:,</TD><TD COLSPAN=4>,' -e "s/^/<TR><TD>$Em/" -e "s,$,</TD></TR>,"
   else
-    zgrep -H -m 1 lessonsLeft $P|grep -v user\.|sed -e 's,/.*#,,' -e "s/^/<TR><TD>$Em/" -e "s, ,</TD><TD>,g" -e "s,$,</TD></TR>," -e "s/=/: /g"
+    zgrep -H -m 1 lessonsLeft "$P"|grep -v user\.|sed -e 's,/.*#,,' -e "s/^/<TR><TD>$Em/" -e "s, ,</TD><TD>,g" -e "s,$,</TD></TR>," -e "s/=/: /g"
   fi
 done
 rm $Midnight

gradint-build/samples/utils/email-lesson.sh → server/email-lesson.sh

@@ -3,114 +3,124 @@
 # email-lesson.sh: a script that can help you to
 # automatically distribute daily Gradint lessons
 # to students using a web server with reminder
-# emails.  Version 1.1127
+# emails.  Version 1.16
 
-# (C) 2007-2010 Silas S. Brown, License: GPL
+# (C) 2007-2010,2020-2022,2024 Silas S. Brown, License: GPL
 
-export DEFAULT_SUBJECT_LINE="Vocabulary practice (automatic message from gradint)"
-export DEFAULT_FORGOT_YESTERDAY="You forgot your lesson yesterday.
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+
+DEFAULT_SUBJECT_LINE="Vocabulary practice (automatic message from gradint)"
+DEFAULT_FORGOT_YESTERDAY="You forgot your lesson yesterday.
 Please remember to download your lesson from"
 # (NB include the words "you forgot" so that it's obvious this is a reminder not an additional lesson)
-export DEFAULT_EXPLAIN_FORGOT="Please try to hear one lesson every day.  If you download that lesson today,
+DEFAULT_EXPLAIN_FORGOT="Please try to hear one lesson every day.  If you download that lesson today,
 this program will make the next one for tomorrow."
-export DEFAULT_NEW_LESSON="Your lesson for today is at"
-export DEFAULT_LISTEN_TODAY="Please download and listen to it today."
-export DEFAULT_AUTO_MESSAGE="This is an automatic message from the gradint program.
+DEFAULT_NEW_LESSON="Your lesson for today is at"
+DEFAULT_LISTEN_TODAY="Please download and listen to it today."
+DEFAULT_AUTO_MESSAGE="This is an automatic message from the gradint program.
 Any problems, requests, or if you no longer wish to receive these emails,
 let me know."
 
-if ! test -e gradint.py; then
+if ! [ -e gradint.py ]; then
   echo "Error: This script should ALWAYS be run in the gradint directory."
   exit 1
 fi
 
-if which mail >/dev/null 2>/dev/null; then export DefaultMailProg=mail
-elif which mutt >/dev/null 2>/dev/null; then export DefaultMailProg="mutt -x"
-else export DefaultMailProg="ssh example.org mail"
+if which mail >/dev/null 2>/dev/null; then DefaultMailProg=mail
+elif which mutt >/dev/null 2>/dev/null; then DefaultMailProg="mutt -x"
+else DefaultMailProg="ssh example.org mail"
 fi
 
-if test "a$1" == "a--run"; then
+if [ "$1" == "--run" ]; then
   set -o pipefail # make sure errors in pipes are reported
-  if ! test -d email_lesson_users; then
+  if ! [ -d email_lesson_users ]; then
     echo "Error: script does not seem to have been set up yet"
     exit 1
   fi
-  export Gradint_Dir=$(pwd)
-  cd email_lesson_users
+  Gradint_Dir=$(pwd)
+  cd email_lesson_users || exit
   . config
-  if test -e $Gradint_Dir/.email-lesson-running; then
-    export Msg="Another email-lesson.sh --run is running - exitting.  (Remove $Gradint_Dir/.email-lesson-running if this isn't the case.)"
+  if [ -e "$Gradint_Dir/.email-lesson-running" ]; then
+    Msg="Another email-lesson.sh --run is running - exitting.  (Remove $Gradint_Dir/.email-lesson-running if this isn't the case.)"
     echo "$Msg"
     echo "$Msg"|$MailProg -s email-lesson-not-running $ADMIN_EMAIL # don't worry about retrying that
     exit 1
   fi
-  touch $Gradint_Dir/.email-lesson-running
-  if echo $PUBLIC_HTML | grep : >/dev/null && man ssh 2>/dev/null | grep ControlMaster >/dev/null; then
+  touch "$Gradint_Dir/.email-lesson-running"
+  if echo "$PUBLIC_HTML" | grep : >/dev/null && man ssh 2>/dev/null | grep ControlMaster >/dev/null; then
     # this version of ssh is new enough to support ControlPath, and PUBLIC_HTML indicates a remote host, so let's do it all through one connection
-    export ControlPath="-o ControlPath=$TMPDIR/__gradint_ctrl"
-    while true; do ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS -n -o ControlMaster=yes $ControlPath $(echo $PUBLIC_HTML|sed -e 's/:.*//') sleep 86400; sleep 10; done & export MasterPid=$!
+    ControlPath="-o ControlPath=$TMPDIR/__gradint_ctrl"
+    while true; do ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS -n -o ControlMaster=yes $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') sleep 86400; sleep 10; done & MasterPid=$!
   else unset MasterPid
   fi
-  (while ! bash -c "$CAT_LOGS_COMMAND"; do echo "cat-logs failed, re-trying in 61 seconds" 1>&2;sleep 61; done) | grep '/user\.' > $TMPDIR/._email_lesson_logs
+  (while ! bash -c "$CAT_LOGS_COMMAND"; do echo "cat-logs failed, re-trying in 61 seconds" >&2;sleep 61; done) | grep '/user\.' > "$TMPDIR/._email_lesson_logs"
   # (note: sleeping odd numbers of seconds so we can tell where it is if it gets stuck in one of these loops)
-  export Users="$(echo user.*)"
+  Users="$(echo user.*)"
   cd ..
   unset NeedRunMirror
   for U in $Users; do
     . email_lesson_users/config
-    if ! test "a$GLOBAL_GRADINT_OPTIONS" == a; then export GLOBAL_GRADINT_OPTIONS="$GLOBAL_GRADINT_OPTIONS ;"; fi
+    if [ "$GLOBAL_GRADINT_OPTIONS" ]; then GLOBAL_GRADINT_OPTIONS="$GLOBAL_GRADINT_OPTIONS ;"; fi
     # set some (but not all!) variables to defaults in case not set in profile
-    export SUBJECT_LINE="$DEFAULT_SUBJECT_LINE"
-    export FORGOT_YESTERDAY="$DEFAULT_FORGOT_YESTERDAY"
-    export LISTEN_TODAY="$DEFAULT_LISTEN_TODAY"
-    export NEW_LESSON="$DEFAULT_NEW_LESSON"
-    export EXPLAIN_FORGOT="$DEFAULT_EXPLAIN_FORGOT"
-    export AUTO_MESSAGE="$DEFAULT_AUTO_MESSAGE"
+    SUBJECT_LINE="$DEFAULT_SUBJECT_LINE"
+    FORGOT_YESTERDAY="$DEFAULT_FORGOT_YESTERDAY"
+    LISTEN_TODAY="$DEFAULT_LISTEN_TODAY"
+    NEW_LESSON="$DEFAULT_NEW_LESSON"
+    EXPLAIN_FORGOT="$DEFAULT_EXPLAIN_FORGOT"
+    AUTO_MESSAGE="$DEFAULT_AUTO_MESSAGE"
     unset Extra_Mailprog_Params1 Extra_Mailprog_Params2 GRADINT_OPTIONS
-    export Use_M3U=no
-    export FILE_TYPE=mp3
-    if grep $'\r' email_lesson_users/$U/profile >/dev/null; then
+    Use_M3U=no
+    FILE_TYPE=mp3
+    if grep $'\r' "email_lesson_users/$U/profile" >/dev/null; then
       # Oops, someone edited profile in a DOS line-endings editor (e.g. Wenlin on WINE for CJK stuff).  DOS line endings can mess up Extra_Mailprog_Params settings.
-      cat email_lesson_users/$U/profile | tr -d $'\r' > email_lesson_users/$U/profile.removeCR
-      mv email_lesson_users/$U/profile.removeCR email_lesson_users/$U/profile
+      tr -d $'\r' < "email_lesson_users/$U/profile" > email_lesson_users/$U/profile.removeCR
+      mv "email_lesson_users/$U/profile.removeCR" "email_lesson_users/$U/profile"
     fi
-    . email_lesson_users/$U/profile
-    if test $Use_M3U == yes; then export FILE_TYPE_2=m3u
-    else export FILE_TYPE_2=$FILE_TYPE; fi
+    . "email_lesson_users/$U/profile"
+    if [ "$Use_M3U" == yes ]; then FILE_TYPE_2=m3u
+    else FILE_TYPE_2=$FILE_TYPE; fi
     if echo "$MailProg" | grep ssh >/dev/null; then
       # ssh discards a level of quoting, so we need to be more careful
-      export SUBJECT_LINE="\"$SUBJECT_LINE\""
-      export Extra_Mailprog_Params1="\"$Extra_Mailprog_Params1\""
-      export Extra_Mailprog_Params2="\"$Extra_Mailprog_Params2\""
+      SUBJECT_LINE="\"$SUBJECT_LINE\""
+      Extra_Mailprog_Params1="\"$Extra_Mailprog_Params1\""
+      Extra_Mailprog_Params2="\"$Extra_Mailprog_Params2\""
     fi
-    if test -e email_lesson_users/$U/lastdate; then
-      if test $(cat email_lesson_users/$U/lastdate) == $(date +%Y%m%d); then
+    if [ -e "email_lesson_users/$U/lastdate" ]; then
+      if [ "$(cat "email_lesson_users/$U/lastdate")" == "$(date +%Y%m%d)" ]; then
         # still on same day - do nothing with this user this time
 	continue
       fi
-      if ! grep $U-$(cat email_lesson_users/$U/lastdate)\. $TMPDIR/._email_lesson_logs >/dev/null
+      if ! grep "$U-$(cat email_lesson_users/$U/lastdate)"\. "$TMPDIR/._email_lesson_logs" >/dev/null
       # (don't add $FILE_TYPE after \. in case it has been changed)
       then
-        export Did_Download=0
-        if test -e email_lesson_users/$U/rollback; then
-          if test -e email_lesson_users/$U/progress.bak; then
-            mv email_lesson_users/$U/progress.bak email_lesson_users/$U/progress.txt
-            rm -f email_lesson_users/$U/progress.bin
-            export Did_Download=1 # (well actually they didn't, but we're rolling back)
+        Did_Download=0
+        if [ -e "email_lesson_users/$U/rollback" ]; then
+          if [ -e "email_lesson_users/$U/progress.bak" ]; then
+            mv "email_lesson_users/$U/progress.bak" "email_lesson_users/$U/progress.txt"
+            rm -f "email_lesson_users/$U/progress.bin"
+            Did_Download=1 # (well actually they didn't, but we're rolling back)
           fi # else can't rollback, as no progress.bak
-          if test -e email_lesson_users/$U/podcasts-to-send.old; then
-            mv email_lesson_users/$U/podcasts-to-send.old email_lesson_users/$U/podcasts-to-send
+          if [ -e "email_lesson_users/$U/podcasts-to-send.old" ]; then
+            mv "email_lesson_users/$U/podcasts-to-send.old" "email_lesson_users/$U/podcasts-to-send"
           fi
         fi
-      else export Did_Download=1; fi
-      rm -f email_lesson_users/$U/rollback
-      if test $Did_Download == 0; then
+      else Did_Download=1; fi
+      rm -f "email_lesson_users/$U/rollback"
+      if [ $Did_Download == 0 ]; then
         # send a reminder
-        export DaysOld="$(python -c "import os,time;print int((time.time()-os.stat('email_lesson_users/$U/lastdate').st_mtime)/3600/24)")"
-        if test $DaysOld -lt 5 || test $(date +%u) == 1; then # (remind only on Mondays if not checked for 5 days, to avoid filling up inboxes when people are away and can't get to email)
-        while ! $MailProg -s "$SUBJECT_LINE" $STUDENT_EMAIL "$Extra_Mailprog_Params1" "$Extra_Mailprog_Params2" <<EOF
+        DaysOld="$(python -c "import os,time;print(int((time.time()-os.stat('email_lesson_users/$U/lastdate').st_mtime)/3600/24))")"
+        if [ $DaysOld -lt 5 ] || [ $(date +%u) == 1 ]; then # (remind only on Mondays if not checked for 5 days, to avoid filling up inboxes when people are away and can't get to email)
+        while ! $MailProg -s "$SUBJECT_LINE" "$STUDENT_EMAIL" "$Extra_Mailprog_Params1" "$Extra_Mailprog_Params2" <<EOF
 $FORGOT_YESTERDAY
-$OUTSIDE_LOCATION/$U-$(cat email_lesson_users/$U/lastdate).$FILE_TYPE_2
+$OUTSIDE_LOCATION/$U-$(cat "email_lesson_users/$U/lastdate").$FILE_TYPE_2
 $EXPLAIN_FORGOT
 
 $AUTO_MESSAGE
@@ -119,68 +129,68 @@ do echo "mail sending failed; retrying in 62 seconds"; sleep 62; done; fi
         continue
       else
         # delete the previous lesson
-        if echo $PUBLIC_HTML | grep : >/dev/null; then ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo $PUBLIC_HTML|sed -e 's/:.*//') rm $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$(cat email_lesson_users/$U/lastdate).*
-        else rm $PUBLIC_HTML/$U-$(cat email_lesson_users/$U/lastdate).*; fi
+        if echo "$PUBLIC_HTML" | grep : >/dev/null; then ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') rm "$(echo "$PUBLIC_HTML"|sed -e 's/[^:]*://')/$U-$(cat "email_lesson_users/$U/lastdate").*"
+        else rm $PUBLIC_HTML/$U-$(cat "email_lesson_users/$U/lastdate").*; fi
 	# (.* because .$FILE_TYPE and possibly .m3u as well)
       fi
     fi
-    export CurDate=$(date +%Y%m%d)
-    if ! test "a$GRADINT_OPTIONS" == a; then export GRADINT_OPTIONS="$GRADINT_OPTIONS ;"; fi
-    if echo $PUBLIC_HTML | grep : >/dev/null; then export OUTDIR=$TMPDIR
-    else export OUTDIR=$PUBLIC_HTML; fi
-    export USER_GRADINT_OPTIONS="$GLOBAL_GRADINT_OPTIONS $GRADINT_OPTIONS samplesDirectory='email_lesson_users/$U/samples'; progressFile='email_lesson_users/$U/progress.txt'; pickledProgressFile='email_lesson_users/$U/progress.bin'; vocabFile='email_lesson_users/$U/vocab.txt';saveLesson='';loadLesson=0;progressFileBackup='email_lesson_users/$U/progress.bak';outputFile="
+    CurDate=$(date +%Y%m%d)
+    if [ "$GRADINT_OPTIONS" ]; then GRADINT_OPTIONS="$GRADINT_OPTIONS ;"; fi
+    if echo "$PUBLIC_HTML" | grep : >/dev/null; then OUTDIR=$TMPDIR
+    else OUTDIR=$PUBLIC_HTML; fi
+    USER_GRADINT_OPTIONS="$GLOBAL_GRADINT_OPTIONS $GRADINT_OPTIONS samplesDirectory='email_lesson_users/$U/samples'; progressFile='email_lesson_users/$U/progress.txt'; pickledProgressFile='email_lesson_users/$U/progress.bin'; vocabFile='email_lesson_users/$U/vocab.txt';saveLesson='';loadLesson=0;progressFileBackup='email_lesson_users/$U/progress.bak';outputFile="
     # (note: we DO keep progressFileBackup, because it can be useful if the server goes down and the MP3's need to be re-generated or something)
     unset Send_Podcast_Instead
-    if test -s email_lesson_users/$U/podcasts-to-send; then
-      export Send_Podcast_Instead="$(head -1 email_lesson_users/$U/podcasts-to-send)"
-      export NumLines=$(echo $(cat email_lesson_users/$U/podcasts-to-send|wc -l)-1|bc)
-      tail -$NumLines email_lesson_users/$U/podcasts-to-send > email_lesson_users/$U/podcasts-to-send2
-      mv email_lesson_users/$U/podcasts-to-send email_lesson_users/$U/podcasts-to-send.old
-      mv email_lesson_users/$U/podcasts-to-send2 email_lesson_users/$U/podcasts-to-send
-      if test $NumLines == 0; then
-        echo $U | $MailProg -s Warning:email-lesson-run-out-of-podcasts $ADMIN_EMAIL
+    if [ -s "email_lesson_users/$U/podcasts-to-send" ]; then
+      Send_Podcast_Instead="$(head -1 email_lesson_users/$U/podcasts-to-send)"
+      NumLines=$[$(cat "email_lesson_users/$U/podcasts-to-send"|wc -l)-1]
+      tail -$NumLines "email_lesson_users/$U/podcasts-to-send" > "email_lesson_users/$U/podcasts-to-send2"
+      mv "email_lesson_users/$U/podcasts-to-send" "email_lesson_users/$U/podcasts-to-send.old"
+      mv "email_lesson_users/$U/podcasts-to-send2" "email_lesson_users/$U/podcasts-to-send"
+      if [ $NumLines == 0 ]; then
+        echo "$U" | $MailProg -s Warning:email-lesson-run-out-of-podcasts $ADMIN_EMAIL
       fi
-    else rm -f email_lesson_users/$U/podcasts-to-send.old # won't be a rollback after this
+    else rm -f "email_lesson_users/$U/podcasts-to-send.old" # won't be a rollback after this
     fi
-    if test $ENCODE_ON_REMOTE_HOST == 1; then
-      export ToSleep=123
-      while ! if test "a$Send_Podcast_Instead" == a; then
-        python gradint.py "$USER_GRADINT_OPTIONS '-.sh'" </dev/null 2>$TMPDIR/__stderr | ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo $PUBLIC_HTML|sed -e 's/:.*//') "mkdir -p $REMOTE_WORKING_DIR; cd $REMOTE_WORKING_DIR; cat > __gradint.sh;chmod +x __gradint.sh;PATH=$SOX_PATH ./__gradint.sh|$ENCODING_COMMAND $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.$FILE_TYPE;rm -f __gradint.sh";
+    if [ "$ENCODE_ON_REMOTE_HOST" == 1 ]; then
+      ToSleep=123
+      while ! if [ ! "$Send_Podcast_Instead" ]; then
+        python gradint.py "$USER_GRADINT_OPTIONS '-.sh'" </dev/null 2>"$TMPDIR/__stderr" | ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') "mkdir -p $REMOTE_WORKING_DIR; cd $REMOTE_WORKING_DIR; cat > __gradint.sh;chmod +x __gradint.sh;PATH=$SOX_PATH ./__gradint.sh|$ENCODING_COMMAND $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.$FILE_TYPE;rm -f __gradint.sh";
       else
-        cd email_lesson_users/$U ; cat "$Send_Podcast_Instead" | ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo $PUBLIC_HTML|sed -e 's/:.*//') "cat > $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.$FILE_TYPE"; cd ../..;
+        cd "email_lesson_users/$U" ; cat "$Send_Podcast_Instead" | ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') "cat > $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.$FILE_TYPE"; cd ../..;
       fi; do
         # (</dev/null so exceptions don't get stuck on 'press enter to continue' to a temp stderr if running from a terminal)
-        $MailProg -s gradint-to-ssh-failed,-will-retry $ADMIN_EMAIL < $TMPDIR/__stderr
+        $MailProg -s gradint-to-ssh-failed,-will-retry $ADMIN_EMAIL < "$TMPDIR/__stderr"
         # (no spaces in subj so no need to decide whether to single or double quote)
         # (don't worry about mail errors - if net is totally down that's ok, admin needs to know if it's a gradint bug causing infinite loop)
-        sleep $ToSleep ; export ToSleep=$(echo $ToSleep*1.5|bc) # (increasing-time retries)
+        sleep $ToSleep ; ToSleep=$[$ToSleep*1.5] # (increasing-time retries)
       done
-      rm $TMPDIR/__stderr
-      if test $Use_M3U == yes; then
-        while ! ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo $PUBLIC_HTML|sed -e 's/:.*//') "echo $OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE > $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.m3u"; do sleep 63; done
+      rm "$TMPDIR/__stderr"
+      if [ "$Use_M3U" == yes ]; then
+        while ! ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') "echo $OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE > $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.m3u"; do sleep 63; done
       fi
-    else
-      if ! test "a$Send_Podcast_Instead" == a; then
-        (cd email_lesson_users/$U ; cat "$Send_Podcast_Instead") > "$OUTDIR/$U-$CurDate.$FILE_TYPE"
+    else # not ENCODE_ON_REMOTE_HOST
+      if [ "$Send_Podcast_Instead" ]; then
+        (cd "email_lesson_users/$U" ; cat "$Send_Podcast_Instead") > "$OUTDIR/$U-$CurDate.$FILE_TYPE"
       elif ! python gradint.py "$USER_GRADINT_OPTIONS '$OUTDIR/$U-$CurDate.$FILE_TYPE'" </dev/null; then
         echo "Errors from gradint itself (not ssh/network); skipping this user."
         echo "Failed on $U, check output " | $MailProg -s gradint-failed $ADMIN_EMAIL
         continue
       fi
-      if test $Use_M3U == yes; then
-        echo $OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE > $OUTDIR/$U-$CurDate.m3u
+      if [ "$Use_M3U" == yes ]; then
+        echo "$OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE" > "$OUTDIR/$U-$CurDate.m3u"
       fi
-      if echo $PUBLIC_HTML | grep : >/dev/null; then
-        while ! scp $ControlPath -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $OUTDIR/$U-$CurDate.* $PUBLIC_HTML/; do
+      if echo "$PUBLIC_HTML" | grep : >/dev/null; then
+        while ! scp $ControlPath -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $OUTDIR/$U-$CurDate.* "$PUBLIC_HTML/"; do
           echo "scp failed; re-trying in 60 seconds"
   	sleep 64
         done
-        rm $OUTDIR/$U-$CurDate.*
+        rm "$OUTDIR/$U-$CurDate".*
       fi
     fi
-    export NeedRunMirror=1
-    if ! test -e email_lesson_users/$U/progress.bak; then touch email_lesson_users/$U/progress.bak; fi # so rollback works after 1st lesson
-    while ! $MailProg -s "$SUBJECT_LINE" $STUDENT_EMAIL "$Extra_Mailprog_Params1" "$Extra_Mailprog_Params2" <<EOF
+    NeedRunMirror=1
+    if ! [ -e "email_lesson_users/$U/progress.bak" ]; then touch "email_lesson_users/$U/progress.bak"; fi # so rollback works after 1st lesson
+    while ! $MailProg -s "$SUBJECT_LINE" "$STUDENT_EMAIL" "$Extra_Mailprog_Params1" "$Extra_Mailprog_Params2" <<EOF
 $NEW_LESSON
 $OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE_2
 $LISTEN_TODAY
@@ -188,27 +198,27 @@ $LISTEN_TODAY
 $AUTO_MESSAGE
 EOF
 do echo "mail sending failed; retrying in 65 seconds"; sleep 65; done
-    echo $CurDate > email_lesson_users/$U/lastdate
+    echo "$CurDate" > "email_lesson_users/$U/lastdate"
     unset AdminNote
-    if test "a$Send_Podcast_Instead" == a; then
-      if test $(zgrep -H -m 1 lessonsLeft email_lesson_users/$U/progress.txt|sed -e 's/.*=//') == 0; then export AdminNote="Note: $U has run out of new words"; fi
-    elif ! test -e email_lesson_users/$U/podcasts-to-send; then export AdminNote="Note: $U has run out of podcasts"; fi
-    if ! test "a$AdminNote" == a; then
-      while ! echo "$AdminNote"|$MailProg -s gradint-user-ran-out $ADMIN_EMAIL; do echo "Mail sending failed; retrying in 67 seconds"; sleep 67; done
+    if [ "$Send_Podcast_Instead" == a ]; then
+      if [ "$(zgrep -H -m 1 lessonsLeft "email_lesson_users/$U/progress.txt"|sed -e 's/.*=//')" == 0 ]; then AdminNote="Note: $U has run out of new words"; fi
+    elif ! [ -e "email_lesson_users/$U/podcasts-to-send" ]; then AdminNote="Note: $U has run out of podcasts"; fi
+    if [ "$AdminNote" ]; then
+      while ! echo "$AdminNote"|$MailProg -s gradint-user-ran-out "$ADMIN_EMAIL"; do echo "Mail sending failed; retrying in 67 seconds"; sleep 67; done
     fi
   done # end of per-user loop
-  if test "a$NeedRunMirror" == "a1" && ! test "a$PUBLIC_HTML_MIRROR_COMMAND" == a; then
+  if [ "$NeedRunMirror" == "1" ] && [ "$PUBLIC_HTML_MIRROR_COMMAND" ]; then
     while ! $PUBLIC_HTML_MIRROR_COMMAND; do
       echo "PUBLIC_HTML_MIRROR_COMMAND failed; retrying in 79 seconds"
-      echo As subject | $MailProg -s "PUBLIC_HTML_MIRROR_COMMAND failed, will retry" $ADMIN_EMAIL || true # ignore errors
+      echo As subject | $MailProg -s "PUBLIC_HTML_MIRROR_COMMAND failed, will retry" "$ADMIN_EMAIL" || true # ignore errors
       sleep 79
     done
   fi
-  rm -f $TMPDIR/._email_lesson_logs
-  if ! test a$MasterPid == a; then
+  rm -f "$TMPDIR/._email_lesson_logs"
+  if [ $MasterPid ] ; then
     kill $MasterPid
-    kill $(ps axwww|grep $TMPDIR/__gradint_ctrl|sed -e 's/^ *//' -e 's/ .*//') 2>/dev/null
-    rm -f $TMPDIR/__gradint_ctrl # in case ssh doesn't
+    kill $(pgrep -f "$TMPDIR/__gradint_ctrl") 2>/dev/null
+    rm -f "$TMPDIR/__gradint_ctrl" # in case ssh doesn't
   fi
   rm -f "$Gradint_Dir/.email-lesson-running"
   exit 0
@@ -217,10 +227,10 @@ fi
 echo "After setting up users, run this script daily with --run on the command line."
 echo "As --run was not specified, it will now go into setup mode."
 # Setup:
-if test "a$EDITOR" == a; then
+if ! [ "$EDITOR" ]; then
   echo "Error: No EDITOR environment variable set"; exit 1
 fi
-if ! test -e email_lesson_users/config; then
+if ! [ -e email_lesson_users/config ]; then
  echo "It seems the email_lesson_users directory is not set up"
  echo "Press Enter to create a new one,
  or Ctrl-C to quit if you're in the wrong directory"
@@ -228,11 +238,11 @@ if ! test -e email_lesson_users/config; then
  mkdir email_lesson_users || exit 1
  cat > email_lesson_users/config <<EOF
 # You need to edit this file.
-export GLOBAL_GRADINT_OPTIONS="" # if set, will be added to all gradint command lines (e.g. to set synthCache if it's not in advanced.txt)
-export MailProg="$DefaultMailProg" # mail, or mutt -x, or ssh some.host mail, or whatever
-export PUBLIC_HTML=~/public_html # where to put files on the WWW.  If it contains a : then scp will be used to copy them there.
-export OUTSIDE_LOCATION=http://$(hostname -f)/~$(whoami) # where they appear from outside
-export CAT_LOGS_COMMAND="false" # Please change this to a command that cats the
+GLOBAL_GRADINT_OPTIONS="" # if set, will be added to all gradint command lines (e.g. to set synthCache if it's not in advanced.txt)
+MailProg="$DefaultMailProg" # mail, or mutt -x, or ssh some.host mail, or whatever
+PUBLIC_HTML=~/public_html # where to put files on the WWW.  If it contains a : then scp will be used to copy them there.
+OUTSIDE_LOCATION=http://$(hostname -f)/~$(whoami) # where they appear from outside
+CAT_LOGS_COMMAND="false" # Please change this to a command that cats the
 # server logs for at least the last 48 hours.  (On some systems you may need
 # to make the script suid root.)  It is used to check that the users have
 # downloads their lessons and remind them if not.
@@ -244,29 +254,29 @@ export CAT_LOGS_COMMAND="false" # Please change this to a command that cats the
 # control connection (\$ControlPath will expand to
 # nothing on systems with old ssh's that don't support this)
 
-export PUBLIC_HTML_EXTRA_SSH_OPTIONS="" # if set and PUBLIC_HTML is on a remote host, these options will be added to all ssh and scp commands to that host - use this for things like specifying an alternative identity file with -i
+PUBLIC_HTML_EXTRA_SSH_OPTIONS="" # if set and PUBLIC_HTML is on a remote host, these options will be added to all ssh and scp commands to that host - use this for things like specifying an alternative identity file with -i
 
-export PUBLIC_HTML_MIRROR_COMMAND="" # if set, will be run after any new lessons are written to PUBLIC_HTML.
+PUBLIC_HTML_MIRROR_COMMAND="" # if set, will be run after any new lessons are written to PUBLIC_HTML.
 # This is for unusual setups where PUBLIC_HTML is not the real public_html directory but some command can be run to mirror its contents to the real one (perhaps on a remote server that cannot take passwordless SSH from here; of course you'd need to set up an alternative way of getting the files across and the log entries back).
 # Note: Do not add >/dev/null or similar redirects to PUBLIC_HTML_MIRROR_COMMAND as some versions of bash will give an error.
 
 export TMPDIR=/tmp # or /dev/shm or whatever
 
-export ENCODE_ON_REMOTE_HOST=0  # if 1, will ssh to the remote host
+ENCODE_ON_REMOTE_HOST=0  # if 1, will ssh to the remote host
 # that's specified in PUBLIC_HTML (which *must* be host:path in this case)
 # and will run an encoding command *there*, instead of encoding
 # locally and copying up.  This is useful if the local machine is the
 # only place gradint can run but it can't encode (e.g. Linux server running on NAS device).
 # If you set the above to 1 then you also need to set these options:
-export REMOTE_WORKING_DIR=. # directory to change to on remote host e.g. /tmp/gradint (will create with mkdir -p if does not exist)
+REMOTE_WORKING_DIR=. # directory to change to on remote host e.g. /tmp/gradint (will create with mkdir -p if does not exist)
 # (make sure $PUBLIC_HTML etc is absolute or is relative to $REMOTE_WORKING_DIR) (don't use spaces in these pathnames)
-export SOX_PATH=$PATH
+SOX_PATH=$PATH
 # make sure the above includes the remote host's "sox" as well as basic commands
-export ENCODING_COMMAND="lame --vbr-new -V 9 -"
+ENCODING_COMMAND="lame --vbr-new -V 9 -"
 # (used only if ENCODE_ON_REMOTE_HOST is set)
 # (include the full path for that if necessary; SOX_PATH will NOT be searched)
 # (set options for encode wav from stdin & output to the file specified on nxt parameter.  No shell quoting.)
-export ADMIN_EMAIL=admin@example.com # to report errors
+ADMIN_EMAIL=admin@example.com # to report errors
 EOF
  cd email_lesson_users; $EDITOR config; cd ..
  echo "Created email_lesson_users/config"
@@ -275,16 +285,16 @@ cd email_lesson_users
 while true; do
   echo "Type a user alias (or just press Enter) to add a new user, or Ctrl-C to quit"
   read Alias
-  export ID=$(mktemp -d user.$(python -c 'import random; print random.random()')XXXXXX) # (newer versions of mktemp allow more than 6 X's so the python step isn't necessary, but just in case we want to make sure that it's hard to guess the ID)
-  if ! test "a$Alias" == a; then ln -s $ID "$Alias"; fi
-  cd $ID
+  ID=$(mktemp -d user.$(python -c 'import random; print(random.random())')XXXXXX) # (newer versions of mktemp allow more than 6 X's so the python step isn't necessary, but just in case we want to make sure that it's hard to guess the ID)
+  if [ "$Alias" ]; then ln -s "$ID" "$Alias"; fi
+  cd "$ID" || exit 1
   cat > profile <<EOF
 # You need to edit the settings in this file.
-export STUDENT_EMAIL=student@example.org  # change to student's email address
+STUDENT_EMAIL=student@example.org  # change to student's email address
 export GRADINT_OPTIONS="" # extra gradint command-line options, for example to
                           # specify a different first and second language
-export FILE_TYPE=mp3 # change to something else if you want
-export Use_M3U=no # if yes, sends a .m3u link to the student
+FILE_TYPE=mp3 # change to something else if you want
+Use_M3U=no # if yes, sends a .m3u link to the student
 # instead of sending the file link directly.  Use this if
 # the student needs to stream over a slow link, but note
 # that it makes offline listening one step more complicated.
@@ -314,33 +324,33 @@ export Use_M3U=no # if yes, sends a .m3u link to the student
 # students that the lessons will now come from a different address.
 
 # Optional settings for customising the text of the message:
-export SUBJECT_LINE="$DEFAULT_SUBJECT_LINE"
-export FORGOT_YESTERDAY="$DEFAULT_FORGOT_YESTERDAY"
-export LISTEN_TODAY="$DEFAULT_LISTEN_TODAY"
-export NEW_LESSON="$DEFAULT_NEW_LESSON"
-export EXPLAIN_FORGOT="$DEFAULT_EXPLAIN_FORGOT"
-export AUTO_MESSAGE="$DEFAULT_AUTO_MESSAGE"
-export Extra_Mailprog_Params1=""
-export Extra_Mailprog_Params2=""
+SUBJECT_LINE="$DEFAULT_SUBJECT_LINE"
+FORGOT_YESTERDAY="$DEFAULT_FORGOT_YESTERDAY"
+LISTEN_TODAY="$DEFAULT_LISTEN_TODAY"
+NEW_LESSON="$DEFAULT_NEW_LESSON"
+EXPLAIN_FORGOT="$DEFAULT_EXPLAIN_FORGOT"
+AUTO_MESSAGE="$DEFAULT_AUTO_MESSAGE"
+Extra_Mailprog_Params1=""
+Extra_Mailprog_Params2=""
 # You may need to set Extra_Mailprog_Params to extra parameters
 # if the subject or text includes characters that need to be sent
-# in a specific charset.  For example, to send in Chinese GB2312
-# with Mutt, you can do this:
+# in a specific charset.  For example, to send Chinese (Simplified)
+# in UTF-8 with Mutt, you can do this:
 # export GRADINT_OPTIONS="firstLanguage='zh'; secondLanguage='en'; otherLanguages=[]"
 # export LANG=C
-# export Extra_Mailprog_Params1="-e"
-# export Extra_Mailprog_Params2="set charset='gb2312'; set send_charset='gb2312'"
-# export SUBJECT_LINE="英文词汇练习 (English vocabulary practice)"
-# export FORGOT_YESTERDAY="你忘记了昨天的课 (you forgot your lesson yesterday).
-# 请记得下载 (please remember to download) :"
-# export EXPLAIN_FORGOT="请试图天天听一课 (please try to hear one lesson every day)
-# 如果你今天下载, 这个软件要明天给你另一个课.
+# Extra_Mailprog_Params1="-e"
+# Extra_Mailprog_Params2="set charset='utf-8'; set send_charset='utf-8'"
+# SUBJECT_LINE="英文词汇练习 (English vocabulary practice)"
+# FORGOT_YESTERDAY="你忘记了昨天的课 (you forgot your lesson yesterday).
+# 请记得下载 (please remember to download) :"
+# EXPLAIN_FORGOT="请试图天天听一课 (please try to hear one lesson every day)
+# 如果你今天下载, 这个软件要明天给你另一个课.
 # (If you download that lesson today,
 # this program will make the next one for tomorrow.)"
-# export NEW_LESSON="今天的课在以下的网址 (your lesson for today is at)"
-# export LISTEN_TODAY="请你今天下载而听 (please download and listen to it today)."
-# export AUTO_MESSAGE="这个电邮是软件写的 (this is an automatic message from the gradint program).
-# 假如你有问题, 请告诉我 (any problems, let me know)."
+# NEW_LESSON="今天的课在以下的网址 (your lesson for today is at)"
+# LISTEN_TODAY="请你今天下载而听 (please download and listen to it today)."
+# AUTO_MESSAGE="这个电邮是软件写的 (this is an automatic message from the gradint program).
+# 假如你有问题, 请告诉我 (any problems, let me know)."
 
 # You can also override *some* of the email_lesson_users/config
 # options on a per-user basis by putting them here,

gradint-build/samples/utils/espeak.cgi → server/espeak.cgi

 #!/usr/bin/env python
+#  (Python 2 or Python 3)
 
 # espeak.cgi - a CGI script for the eSpeak speech synthesizer
 
-# (c) 2008 Silas S. Brown, License: GPL
-version="1.1211"
+# (c) 2008,2011,2020 Silas S. Brown, License: GPL
+version="1.3"
+
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
 
 # With most webservers you should be able to put this
 # in your public_html and do chmod +x.  You will also need to
@@ -21,15 +32,28 @@ max_input_size = 1000
 
 minSpeed, defaultSpeed, maxSpeed, speedStep = 80,170,370,30 # (NB check defaultSpeed=minSpeed+integer*speedStep)
 
-import os,commands,re,sys
-if commands.getoutput("which espeak 2>/dev/null"): prog="espeak"
-elif commands.getoutput("which speak 2>/dev/null"): prog="speak"
+import os,re,sys
+try: from commands import getoutput # Python 2
+except: from subprocess import getoutput # Python 3
+if getoutput("which espeak 2>/dev/null"): prog="espeak"
+elif getoutput("which speak 2>/dev/null"): prog="speak"
 else: raise Exception("Cannot find espeak")
 
-lang = f.getfirst("l",default_language)
+def S(x):
+    if type("")==type(u""): # Python 3
+        try: return x.decode('utf-8') # in case byte-string
+        except: pass
+    return x
+lang = S(f.getfirst("l",default_language))
 if len(lang)>10 or not re.match("^[a-z0-9-+]*$",lang): lang=default_language
 
-voiceDir=os.environ.get("ESPEAK_DATA_PATH","/usr/share/espeak-data")+"/voices"
+if "ESPEAK_DATA_PATH" in os.environ: voiceDir = os.environ["ESPEAK_DATA_PATH"]
+elif os.path.exists("/usr/share/espeak-data"): voiceDir = "/usr/share/espeak-data"
+elif os.path.exists("/usr/lib/x86_64-linux-gnu/espeak-data"): voiceDir = "/usr/lib/x86_64-linux-gnu/espeak-data"
+else:
+    print ("Content-type: text-plain\n\nUnable to find ESPEAK_DATA_PATH")
+    raise SystemExit
+voiceDir += "/voices"
 
 variants = os.listdir(voiceDir+"/!v")
 if "whisper" in variants and "wisper" in variants: variants.remove("wisper")
@@ -45,16 +69,18 @@ try: speed=int(speed)
 except: speed=defaultSpeed
 if speed<minSpeed or speed>maxSpeed: speed=defaultSpeed
 
-t = f.getfirst("t","")
+t = S(f.getfirst("t",""))
 if len(t)>max_input_size: t=""
 else:
-    try: t.decode('utf-8')
-    except: t="" # not valid utf-8
+    if not type(t)==type(u""):
+        try: t.decode('utf-8')
+        except: t="" # not valid utf-8
     if chr(0) in t: t="" # just in case
 
 if len(t)>stream_if_input_bigger_than:
     # streaming - will need sox to convert
-    if not commands.getoutput("which sox 2>/dev/null"): raise Exception("Cannot find sox")
+    if not getoutput("which sox 2>/dev/null"): raise Exception("Cannot find sox")
+    fname=None
 else:
     # not streaming (so can fill in length etc) - will need a writable file in a private tmp directory, preferably in memory
     worked = 0
@@ -69,15 +95,16 @@ else:
     open(fname2,"w") # raising exception if it's unwritable (try changing to a suitable directory)
 
 # in case espeak can't find a utf-8 locale
-loc=commands.getoutput("locale -a|grep -i 'utf-*8$'|head -1").strip()
+loc=getoutput("locale -a|grep -i 'utf-*8$'|head -1").strip()
 if loc: os.environ["LC_CTYPE"]=loc
 
 def getName(f):
+    f = S(f)
     o=open(f)
     line=""
     for t in range(10):
-        line=o.readline()
-        if "name" in line: return line.split()[1]
+        line=getBuf(o).readline()
+        if u"name".encode('latin1') in line: return S(line.split()[1])
     return f[f.rindex("/")+1:] # assumes it'll be a full pathname
 def isDirectory(directory):
     oldDir = os.getcwd()
@@ -88,10 +115,21 @@ def isDirectory(directory):
     os.chdir(oldDir)
     return ret
 
+def getBuf(f):
+    if hasattr(f,"buffer"): return f.buffer # Python 3
+    else: return f # Python 2
+def doPipe(P,t):
+    if type("")==type(u""): # Python 3
+        P = os.popen(P,"w")
+        if type(t)==type(u""): P.write(t)
+        else: P.buffer.write(t)
+        P.close()
+    else: os.popen(P,"wb").write(t) # Python 2
+
 if t and f.getfirst("qx",""):
     sys.stdout.write("Content-Type: text/plain; charset=utf-8\n\n")
     sys.stdout.flush() # help mathopd
-    os.popen(prog+" -v "+lang+" -q -X -m 2>/dev/null","wb").write(t)
+    doPipe(prog+" -v "+lang+" -q -X -m 2>/dev/null",t)
 elif t:
     prog_with_params = prog+" -v "+lang+"+"+variant+" -s "+str(speed)+" -m"
     # TODO -p 0-99 default 50 (pitch adjustment)
@@ -100,19 +138,20 @@ elif t:
         sys.stdout.write("Content-Type: audio/basic\nContent-Disposition: attachment; filename=\""+t+"_"+lang+".au\"\n\n") # using .au instead of .wav because Windows Media Player doesn't like incorrect length fields in wav.  And make sure it's attachment otherwise Mac OS QuickTime etc can have problems when server is slow
         # problem is, WILL NEED CONVERTING for gradint (unless want to use "sox" on the Windows version before playing via winsound) (but the espeak no-length wav files will probably be wrong on that anyway).  Should be OK because we're doing this only in the case of len(t)>stream_if_input_bigger_than.
         sys.stdout.flush() # help mathopd
-        os.popen(prog_with_params+" --stdout 2>/dev/null | sox -t wav - -t au - 2>/dev/null","wb").write(t)
-    else:
-        os.popen(prog_with_params+" -w "+fname2+" 2>/dev/null","wb").write(t)
+        doPipe(prog_with_params+" --stdout 2>/dev/null | sox -t wav - -t au - 2>/dev/null",t)
+    else: # not streaming
+        doPipe(prog_with_params+" -w "+fname2+" 2>/dev/null",t)
         sys.stdout.write("Content-Type: audio/wav\nContent-Disposition: attachment; filename=\""+t+"_"+lang+".wav\"\n\n")
-        sys.stdout.write(open(fname2,"rb").read())
+        sys.stdout.flush()
+        getBuf(sys.stdout).write(open(fname2,"rb").read())
 else:
-    sys.stdout.write("Content-Type: text/html; charset=utf-8\n\n<HTML><BODY>") # (specify utf-8 here in case accept-charset is not recognised, e.g. some versions of IE6)
-    banner = commands.getoutput(prog+" --help|head -3").strip()
+    sys.stdout.write('Content-Type: text/html; charset=utf-8\n\n<HTML><head><meta name="viewport" content="width=device-width"></head><BODY>') # (specify utf-8 here in case accept-charset is not recognised, e.g. some versions of IE6)
+    banner = S(getoutput(prog+" --help|head -3").strip())
     sys.stdout.write("This is espeak.cgi version "+version+", using <A HREF=http://espeak.sourceforge.net/>eSpeak</A> "+" ".join(banner.split()[1:]))
     if not loc: sys.stdout.write("<br>Warning: could not find a UTF-8 locale; espeak may malfunction on some languages")
-    warnings=commands.getoutput(prog+" -q -x .").strip() # make sure any warnings about locales are output
+    warnings=S(getoutput(prog+" -q -x .").strip()) # make sure any warnings about locales are output
     if warnings: sys.stdout.write("<br>"+warnings)
-    sys.stdout.write("<FORM accept-charset=UTF-8>Text or SSML: <INPUT TYPE=text NAME=t STYLE='width:80%'><br>Language: <SELECT NAME=l>")
+    sys.stdout.write("<FORM method=post accept-charset=UTF-8>Text or SSML: <INPUT TYPE=text NAME=t STYLE='width:80%'><br>Language: <SELECT NAME=l>")
     ld=os.listdir(voiceDir)
     directories = {}
     for f in ld[:]:
@@ -137,9 +176,9 @@ else:
         if v==variant: sys.stdout.write(" SELECTED")
         sys.stdout.write(">"+name+"</OPTION>")
     sys.stdout.write("</SELECT> Speed: <SELECT NAME=s>")
-    for ss in range(minSpeed,maxSpeed,speedStep)+[maxSpeed]:
+    for ss in list(range(minSpeed,maxSpeed,speedStep))+[maxSpeed]:
         sys.stdout.write("<OPTION VALUE="+str(ss))
         if ss==speed: sys.stdout.write(" SELECTED")
         sys.stdout.write(">"+str(ss)+"</OPTION>")
     sys.stdout.write("</SELECT> <INPUT TYPE=submit NAME=qx VALUE=\"View phonemes\"><center><big><INPUT TYPE=submit VALUE=SPEAK></big></center></FORM></BODY></HTML>")
-os.system("rm -rf \""+fname+"\"") # clean up temp dir
+if fname: os.system("rm -rf \""+fname+"\"") # clean up temp dir

server/gradint.cgi

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#  (either Python 2 or Python 3)
+
+program_name = "gradint.cgi v1.38 (c) 2011,2015,2017-25 Silas S. Brown.  GPL v3+"
+
+#    This program is free software; you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation; either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+
+gradint_dir = "$HOME/gradint" # include samples/prompts
+path_add = "$HOME/gradint/bin" # include sox, lame, espeak, maybe oggenc
+lib_path_add = "$HOME/gradint/lib"
+espeak_data_path = "$HOME/gradint"
+
+import os, os.path, sys, cgi, urllib, time, re
+import tempfile, getpass
+myTmp = tempfile.gettempdir()+os.sep+getpass.getuser()+"-gradint-cgi"
+try: from commands import getoutput # Python 2
+except: from subprocess import getoutput # Python 3
+try: from urllib import quote,quote_plus,unquote # Python 2
+except: from urllib.parse import quote,quote_plus,unquote # Python 3
+try: from importlib import reload # Python 3
+except: pass
+home = os.environ.get("HOME","")
+if not home:
+  try:
+    import pwd
+    home = os.path.expanduser("~{0}".format(pwd.getpwuid(os.getuid())[0]))
+  except: home=0
+  if not home: home = ".." # assume we're in public_html
+gradint_dir = gradint_dir.replace("$HOME",home)
+path_add = path_add.replace("$HOME",home)
+lib_path_add = lib_path_add.replace("$HOME",home)
+espeak_data_path = espeak_data_path.replace("$HOME",home)
+try: import Cookie # Python 2
+except: import http.cookies as Cookie # Python 3
+import random
+if "QUERY_STRING" in os.environ and "&" in os.environ["QUERY_STRING"] and ";" in os.environ["QUERY_STRING"]: os.environ["QUERY_STRING"]=os.environ["QUERY_STRING"].replace(";","%3B") # for dictionary sites to add words that contain semicolon
+try: query = cgi.FieldStorage(encoding="utf-8") # Python 3
+except: query = cgi.FieldStorage() # Python 2
+os.chdir(gradint_dir) ; sys.path.insert(0,os.getcwd())
+os.environ["PATH"] = path_add+":"+os.environ["PATH"]
+if "LD_LIBRARY_PATH" in os.environ: os.environ["LD_LIBRARY_PATH"] = lib_path_add+":"+os.environ["LD_LIBRARY_PATH"]
+else: os.environ["LD_LIBRARY_PATH"] = lib_path_add
+os.environ["ESPEAK_DATA_PATH"] = espeak_data_path
+
+cginame = os.sep+sys.argv[0] ; cginame=cginame[cginame.rindex(os.sep)+1:]
+sys.stderr=open("/dev/null","w") ; sys.argv = []
+gradint = None
+def reinit_gradint(): # if calling again, also redo setup_userID after
+    global gradint,langFullName
+    if gradint:
+      if sys.version_info[0]>2: gradint.map,gradint.filter,gradint.chr=gradint._map,gradint._filter,gradint.unichr # undo Python 3 workaround in preparation for it to be done again, because reload doesn't do this (at least not on all Python versions)
+      gradint = reload(gradint)
+    else: import gradint
+    gradint.waitOnMessage = lambda *args:False
+    langFullName = {}
+    for l in gradint.ESpeakSynth().describe_supported_languages().split():
+        abbr,name = gradint.S(l).split("=")
+        langFullName[abbr]=name.replace("_","-")
+    # Try to work out probable default language:
+    lang = os.environ.get("HTTP_ACCEPT_LANGUAGE","")
+    if lang:
+        for c in [',',';','-']:
+            if c in lang: lang=lang[:lang.index(c)]
+        if not lang in langFullName: lang=""
+    global noGTranslate
+    if lang:
+        gradint.firstLanguage = lang
+        if lang=="en": noGTranslate = True
+        else:
+          gradint.secondLanguage="en" # (most probable default)
+          noGTranslate = lang in gradint.GUI_translations # (unless perhaps any are incomplete)
+    elif " zh-" in os.environ.get("HTTP_USER_AGENT",""): # Chinese iPhone w/out Accept-Language
+      gradint.firstLanguage,gradint.secondLanguage = "zh","en"
+      noGTranslate = True # (don't know if it even pops up on that browser, but anyway)
+
+reinit_gradint()
+
+def main():
+  if "id" in query: # e.g. from redirectHomeKeepCookie
+    queryID = query.getfirst("id")
+    if not re.match("[A-Za-z0-9_.-]",queryID): return htmlOut("Bad query.  Bad, bad query.") # to avoid cluttering the disk if we're being given random queries by an attacker.  IDs we generate are numeric only, but allow alphanumeric in case server admin wants to generate them.  Don't allow =, parens, etc (likely random SQL query)
+    os.environ["HTTP_COOKIE"]="id="+queryID
+    print ('Set-Cookie: id=' + queryID+'; expires=Wed, 1 Dec 2036 23:59:59 GMT') # TODO: S2G
+  if has_userID(): setup_userID() # always, even for justSynth, as it may include a voice selection (TODO consequently being called twice in many circumstances, could make this more efficient)
+  filetype=""
+  if "filetype" in query: filetype=query.getfirst("filetype")
+  if not filetype in ["mp3","ogg","wav"]: filetype="mp3"
+  for k in query.keys():
+    if k.startswith("del-"):
+     k=unquote(unquote(k)) # might be needed
+     if '=' in k:
+       l2,l1 = k[4:].split('=')
+       setup_userID()
+       gradint.delOrReplace(gradint.ensure_unicode(l2),gradint.ensure_unicode(l1),"","","delete")
+       return listVocab(True)
+  if "js" in query: # just synthesize (js=text jsl=language)
+    if "jsl" in query: justSynth(query.getfirst("js"), query.getfirst("jsl"),filetype=filetype)
+    else: justSynth(query.getfirst("js"),filetype=filetype)
+  elif "spk" in query: # speak (l1,l2 the langs, l1w,l2w the words)
+    gradint.justSynthesize="0"
+    if "l2w" in query and query.getfirst("l2w"):
+      gradint.startBrowser=lambda *args:0
+      if query.getfirst("l2")=="zh" and gradint.generalCheck(query.getfirst("l2w"),"zh"): gradint.justSynthesize += "#en Pinyin needs tones.  Please go back and add tone numbers." # speaking it because alert box might not work and we might be being called from HTML5 Audio stuff (TODO maybe duplicate generalCheck in js, if so don't call HTML5 audio, then we can have an on-screen message here)
+      else: gradint.justSynthesize += "#"+query.getfirst("l2").replace("#","").replace('"','')+" "+query.getfirst("l2w").replace("#","").replace('"','')
+    if "l1w" in query and query.getfirst("l1w"): gradint.justSynthesize += "#"+query.getfirst("l1").replace("#","").replace('"','')+" "+query.getfirst("l1w").replace("#","").replace('"','')
+    if gradint.justSynthesize=="0": return htmlOut(withLocalise('You must type a word in the box before pressing the Speak button.')+backLink) # TODO maybe add a Javascript test to the form also, IF can figure out if window.alert works
+    serveAudio(stream = len(gradint.justSynthesize)>100, filetype=filetype)
+  elif "add" in query: # add to vocab (l1,l2 the langs, l1w,l2w the words)
+    if "l2w" in query and query.getfirst("l2w") and "l1w" in query and query.getfirst("l1w"):
+      gradint.startBrowser=lambda *args:0
+      if query.getfirst("l2")=="zh": gcmsg=gradint.generalCheck(query.getfirst("l2w"),"zh")
+      else: gcmsg=None
+      if gcmsg: htmlOut(gradint.B(gcmsg)+gradint.B(backLink))
+      else: addWord(query.getfirst("l1w"),query.getfirst("l2w"),query.getfirst("l1"),query.getfirst("l2"))
+    else: htmlOut(withLocalise('You must type words in both boxes before pressing the Add button.')+backLink) # TODO maybe add a Javascript test to the form also, IF can figure out a way to tell whether window.alert() works or not
+  elif "bulkadd" in query: # bulk adding, from authoring options
+    dirID = setup_userID()
+    def isOK(x):
+      if x[0]=='W':
+        try:
+          int(x[1:])
+          return True
+        except: pass
+    def mycmp(x,y): return cmp(int(x[1:]),int(y[1:]))
+    keyList = sorted(filter(lambda x:isOK(x),query.keys()),mycmp)
+    for k in keyList:
+      l2w,l1w = query.getfirst(k).split('=',1)
+      addWord(l1w,l2w,query.getfirst("l1"),query.getfirst("l2"),False)
+    redirectHomeKeepCookie(dirID,"&dictionary=1") # '1' is special value for JS-only back link; don't try to link to referer as it might be a generated page
+  elif "clang" in query: # change languages (l1,l2)
+    dirID = setup_userID()
+    if (gradint.firstLanguage,gradint.secondLanguage) == (query.getfirst("l1"),query.getfirst("l2")) and not query.getfirst("clang")=="ignore-unchanged": return htmlOut(withLocalise('You must change the settings before pressing the Change Languages button.')+backLink) # (external scripts can set clang=ignore-unchanged)
+    gradint.updateSettingsFile(gradint.settingsFile,{"firstLanguage": query.getfirst("l1"),"secondLanguage":query.getfirst("l2")})
+    redirectHomeKeepCookie(dirID)
+  elif "swaplang" in query: # swap languages
+    dirID = setup_userID()
+    gradint.updateSettingsFile(gradint.settingsFile,{"firstLanguage": gradint.secondLanguage,"secondLanguage":gradint.firstLanguage})
+    redirectHomeKeepCookie(dirID)
+  elif "editsave" in query: # save 'vocab'
+    dirID = setup_userID()
+    if "vocab" in query: vocab=query.getfirst("vocab")
+    else: vocab="" # user blanked it
+    open(gradint.vocabFile,"w").write(vocab)
+    redirectHomeKeepCookie(dirID)
+  elif "edit" in query: # show the edit form
+    dirID = setup_userID()
+    try: v=open(gradint.vocabFile).read()
+    except: v="" # (shouldn't get here unless they hack URLs)
+    htmlOut('<form action="'+cginame+'" method="post"><textarea name="vocab" style="width:100%;height:80%" rows="15" cols="50">'+v+'</textarea><br><input type=submit name=editsave value="Save changes"> | <input type=submit name=placeholder value="Cancel"></form>',"Text edit your vocab list")
+  elif "lesson" in query: # make lesson ("Start lesson" button)
+    setup_userID()
+    gradint.maxNewWords = int(query.getfirst("new")) # (shouldn't need sensible-range check here if got a dropdown; if they really want to hack the URL then ok...)
+    gradint.maxLenOfLesson = int(float(query.getfirst("mins"))*60)
+    # TODO save those settings for next time also?
+    serveAudio(stream = True, inURL = False, filetype=filetype)
+  elif "bigger" in query or "smaller" in query:
+    u = setup_userID() ; global zoom
+    if "bigger" in query: zoom = int(zoom*1.1)
+    else: zoom = int(zoom/1.1 + 0.5)
+    open(u+"-zoom.txt","w").write("%d\n" % zoom)
+    listVocab(True)
+  elif any("variant"+str(c) in query for c in range(max(len(gradint.GUI_translations[v]) for v in gradint.GUI_translations.keys() if v.startswith("@variants-")))):
+    for c in range(max(len(gradint.GUI_translations[v]) for v in gradint.GUI_translations.keys() if v.startswith("@variants-"))): #TODO duplicate code
+      if "variant"+str(c) in query: break
+    u = setup_userID()
+    gradint.updateSettingsFile(u+"-settings.txt",{"scriptVariants":{gradint.GUI_languages.get(gradint.firstLanguage,gradint.firstLanguage):c}})
+    setup_userID() ; listVocab(True)
+  elif "voNormal" in query: # voice option = normal
+    setup_userID()
+    gradint.voiceOption=""
+    gradint.updateSettingsFile(gradint.settingsFile,{"voiceOption":""})
+    listVocab(True)
+  elif "vopt" in query: # set voice option
+    setup_userID()
+    for v in gradint.guiVoiceOptions:
+      if v.lower()=="-"+query.getfirst("vopt").lower():
+        gradint.voiceOption = v
+        gradint.updateSettingsFile(gradint.settingsFile,{"voiceOption":v})
+        break
+    listVocab(True)
+  elif "lFinish" in query:
+    dirID = setup_userID()
+    try: os.rename(gradint.progressFile+'-new',gradint.progressFile)
+    except: pass # probably a duplicate GET
+    try: os.remove(gradint.progressFile+'-ts') # the timestamp file
+    except: pass
+    redirectHomeKeepCookie(dirID)
+  elif not isAuthoringOption(query): listVocab(has_userID()) # default screen
+
+def U(x):
+  try: return x.decode('utf-8')
+  except: return x
+
+def isAuthoringOption(query):
+  # TODO document the ?author=1 option
+  if "author" in query:
+    htmlOut('<form action="'+cginame+'" method="post"><h2>Gradint word list authoring mode</h2>This can help you put word lists on your website. The words will be linked to this Gradint server so your visitors can choose which ones to hear and/or add to their personal lists.<p>Type any text in the box below; use blank lines to separate paragraphs. To embed a word list in your text, type:<br><em>phrase 1</em>=<em>meaning 1</em><br><em>phrase 2</em>=<em>meaning 2</em><br><em>phrase 3</em>=<em>meaning 3</em><br>etc, and <b>make sure there is a blank line before and after the list</b>. Then press <input type=submit name="generate" value="Generate HTML">.<p>Language for phrases: '+langSelect('l2',gradint.secondLanguage)+' and for meanings: '+langSelect('l1',gradint.firstLanguage)+'<p><textarea name="text" style="width:100%;height:80%" rows="15" cols="50"></textarea><br><input type=submit name="generate" value="Generate HTML"></form>',"Word list authoring",links=0)
+    # TODO maybe langSelect for mand+cant together ? (but many wordlists wld be topolect-specific)
+  elif "generate" in query:
+    l1,l2,txt = query.getfirst("l1"),query.getfirst("l2"),query.getfirst("text")
+    paras = "\n".join([l.strip() for l in U(txt).replace("\r\n","\n").replace("\r","\n").split("\n")]).split("\n\n")
+    need_h5a = False
+    for i in xrange(len(paras)):
+        lines = filter(lambda x:x,paras[i].split("\n")) # filter needed for trailing newline on document
+        if allLinesHaveEquals(lines):
+            paras[i] = authorWordList(lines,l1,l2)
+            need_h5a = True
+        # TODO else some wiki markup for links etc ? (but you can alter the HTML after)
+    if need_h5a: h5astr = h5a()
+    else: h5astr = ""
+    htmlOut(HTML_and_preview(h5astr+encodeAmp('<p>'.join(paras))),"HTML result",links=0)
+  else: return False
+  return True
+def allLinesHaveEquals(lines):
+    if not lines: return False
+    for l in lines:
+        if not '=' in l: return False
+    return True
+gradintUrl = os.environ.get("SCRIPT_URI","") # will be http:// or https:// as appropriate
+if not gradintUrl and all(x in os.environ for x in ["REQUEST_SCHEME","SERVER_NAME","SCRIPT_NAME"]): gradintUrl = os.environ["REQUEST_SCHEME"]+"://"+os.environ["SERVER_NAME"]+os.environ["SCRIPT_NAME"]
+if not gradintUrl: gradintUrl = "gradint.cgi" # guessing
+def authorWordList(lines,l1,l2):
+    r=[] ; count = 0
+    # could have target="gradint" in the following, but it may be in a background tab (target="_blank" not recommended as could accumulate many)
+    r.append('<form action="%s" method="post" accept-charset="utf-8"><table style="margin-left:auto;margin-right:auto;border:thin solid blue"><tr><td colspan=3 style="text-align:center"><em>Click on each word for audio</em></td></tr>' % gradintUrl)
+    for l in lines:
+        l2w,l1w = l.split('=',1)
+        r.append('<tr class="notranslate"><td><input type="checkbox" name="W%d" value="%s=%s" checked></td><td>%s</td><td>%s</td></tr>' % (count,l2w,l1w,U(justsynthLink(l2w.encode('utf-8'),l2)).replace('HREF="'+cginame+'?','HREF="'+gradintUrl+'?'),U(justsynthLink(l1w.encode('utf-8'),l1)).replace('HREF="'+cginame+'?','HREF="'+gradintUrl+'?')))
+        count += 1
+    # could have target="gradint" in the following href, but see comment above
+    r.append('<tr><td colspan=3><input type="submit" name="bulkadd" value="Add selected words"> to your <a href="%s">personal list</a></td></tr></table><input type="hidden" name="l1" value="%s"><input type="hidden" name="l2" value="%s"></form>' % (gradintUrl,l1,l2))
+    return ''.join(r)
+def encodeAmp(uniStr):
+  # HTML-ampersand encode when we don't know if the server will be utf-8 after copy/paste
+  r=[]
+  for c in uniStr:
+    if ord(c)>126: r.append("&#"+str(ord(c))+";")
+    else: r.append(c)
+  return ''.join(r)
+def HTML_and_preview(code): return '<h2>HTML code</h2><textarea style="width:100%%;height:40%%" rows=7 cols=50>%s</textarea><h2>Preview</h2>%s' % (code.replace('&','&amp;').replace('<','&lt;'),code)
+
+def justSynth(text,lang="",filetype=""):
+  if lang: lang = lang.replace("#","").replace('"','')+" "
+  gradint.justSynthesize=lang+text.replace("#","").replace('"','')
+  if not filetype in ["mp3","ogg","wav"]: filetype="mp3"
+  serveAudio(stream = len(text)>80, filetype=filetype)
+
+def justsynthLink(text,lang=""): # assumes written function h5a
+  if lang in gradint.synth_partials_voices and gradint.guiVoiceOptions: cacheInfo="&curVopt="+gradint.voiceOption
+  else: cacheInfo=""
+  return '<A HREF="'+cginame+'?js='+gradint.S(quote_plus(text))+'&jsl='+quote_plus(lang)+cacheInfo+'" onClick="return h5a(this);">'+gradint.S(text)+'</A>'
+# TODO if h5a's canPlayType etc works, cld o/p a lesson as a JS web page that does its own 'take out of event stream' and 'progress write-back'.  wld need to code that HERE by inspecting the finished Lesson object, don't call play().
+
+zoom = 100 # in case browser device lacks a zoom UI, we'll provide one
+noGTranslate = False
+def htmlOut(body_u8,title_extra="",links=1):
+    if noGTranslate: print ("Google: notranslate")
+    print ("Content-type: text/html; charset=utf-8\n")
+    if title_extra: title_extra=": "+title_extra
+    print ('<html lang="en"><head><title>Gradint Web edition'+title_extra+'</title>')
+    print ('<meta name="mobileoptimized" content="0"><meta name="viewport" content="width=device-width">')
+    print ('<script>if(window.matchMedia && window.matchMedia("(prefers-color-scheme: dark)").matches)document.write("<style>body,input,textarea { background-color: black; color: #c0c000; } select,input[type=submit],input[type=button] { background-color: #300020; color: #c0c000; } input[type=text] { border: grey groove; } select[disabled],input[disabled] { background-color: #101010; color: #b0b000; } a:link { color: #00b000; } a:visited { color: #00c0c0; } a:hover { color: red; }</style>");</script>')
+    if not zoom==100: print('<style>body{font-size:%d%%;}body *{font-size:100%%;}</style>' % zoom)
+    print ('</head><body>')
+    if type(body_u8)==type(u""): body_u8=body_u8.encode('utf-8')
+    if hasattr(sys.stdout,'buffer'): # Python 3
+      sys.stdout.flush()
+      sys.stdout.buffer.write(body_u8)
+      sys.stdout.flush()
+    else: print(body_u8)
+    print ('<HR>')
+    if links:
+        print ('This is Gradint Web edition.  If you need recorded words or additional functions, please <A HREF="//ssb22.user.srcf.net/gradint/">download the full version of Gradint</A>.')
+        # TODO @ low-priority: Android 3 <input type="file" accept="audio/*;capture=microphone"></input>
+    print ('<p>'+program_name[:program_name.index("(")]+"using "+gradint.program_name[:gradint.program_name.index("(")])
+    print ("</body></html>")
+backLink = ' <A HREF="'+cginame+'" onClick="history.go(-1);return false">Back</A>' # TODO may want to add a random= to the non-js HREF
+
+def serveAudio(stream=0, filetype="mp3", inURL=1):
+  # caller imports gradint (and sets justSynthesize or whatever) first
+  if os.environ.get("HTTP_IF_MODIFIED_SINCE",""):
+    print ("Status: 304 Not Modified\n\n") ; return
+  httpRange = re.match("bytes=([0-9]*)-([0-9]*)$",os.environ.get('HTTP_RANGE','')) # we MUST support Range: for some iOS players (Apple did not follow the HTTP standard of having a sensible fallback if servers respond with 200, and Apache will not do Range for us if we're CGI).  Single Range should be sufficient.
+  if httpRange: httpRange = httpRange.groups()
+  if httpRange==('',''): httpRange = None # must spec one
+  if httpRange:
+    if not httpRange[0]: httpRange=[-int(httpRange[1]),None]
+    elif not httpRange[1]: httpRange=[int(httpRange[0]),None]
+    else: httpRange=[int(httpRange[0]),int(httpRange[1])+1]
+    print ("Status: 206 Partial Content")
+    stream = 0
+  if filetype=="mp3": print ("Content-type: audio/mpeg")
+  else: print ("Content-type: audio/"+filetype) # ok for ogg, wav?
+  if inURL:
+    print ("Last-Modified: Sun, 06 Jul 2008 13:20:05 GMT")
+    print ("Expires: Wed, 1 Dec 2036 23:59:59 GMT") # TODO: S2G
+  print ("Content-disposition: attachment; filename=gradint."+filetype) # helps with some browsers that can't really do streaming
+  gradint.out_type = filetype
+  gradint.waitBeforeStart = 0
+  def mainOrSynth():
+    oldProgress = None ; rollback = False
+    if not gradint.justSynthesize and 'h5a' in query:
+      try: oldProgress = open(gradint.progressFile,'rb').read()
+      except: pass
+      rollback = True
+      if "lesson" in query: random.seed(query.getfirst("lesson")) # so clients that re-GET same lesson from partway through can work
+    try: gradint.main()
+    except SystemExit:
+      if not gradint.justSynthesize:
+        o1,o2 = gradint.write_to_stdout,gradint.outputFile
+        reinit_gradint() ; setup_userID()
+        gradint.write_to_stdout,gradint.outputFile = o1,o2
+        gradint.setSoundCollector(gradint.SoundCollector())
+        gradint.justSynthesize = "en Problem generating the lesson. Check we have prompts for those languages." ; gradint.main()
+        if oldProgress: open(gradint.progressFile,'wb').write(oldProgress)
+        rollback = oldProgress = None
+    if rollback: # roll back pending lFinish
+      os.rename(gradint.progressFile,gradint.progressFile+'-new')
+      if oldProgress: open(gradint.progressFile,'wb').write(oldProgress)
+    # end of def mainOrSynth
+  if stream:
+    print ("")
+    sys.stdout.flush()
+    gradint.write_to_stdout = 1
+    gradint.outputFile="-."+filetype ; gradint.setSoundCollector(gradint.SoundCollector())
+    mainOrSynth()
+  else:
+    gradint.write_to_stdout = 0
+    tempdir = tempfile.mkdtemp()
+    fn,fn2 = tempdir+"/I."+filetype, tempdir+"/O."+filetype
+    if httpRange and "lesson" in query: # try to cache it
+      try: os.mkdir(myTmp)
+      except: pass # exist ok
+      for f in os.listdir(myTmp):
+        if os.stat(myTmp+os.sep+f).st_mtime < time.time()-4000:
+          os.remove(myTmp+os.sep+f)
+      fn = gradint.outputPrefix+str(int(query.getfirst("lesson")))+"."+filetype # (don't be tricked into clobbering paths with non-int lesson IDs)
+    if not os.path.exists(fn):
+      gradint.outputFile=fn
+      gradint.setSoundCollector(gradint.SoundCollector())
+      mainOrSynth()
+    if httpRange:
+      total = os.stat(fn).st_size
+      open(fn2,"wb").write(open(fn,"rb").read()[httpRange[0]:httpRange[1]])
+      if httpRange[0]<0: httpRange[0] += total
+      if not httpRange[1]: httpRange[1] = total
+      print("Content-Range: bytes %d-%d/%d" % (httpRange[0],httpRange[1]-1,total))
+    else: fn2 = fn
+    print ("Content-Length: "+repr(os.stat(fn2).st_size)+"\n")
+    sys.stdout.flush()
+    os.system("cat "+fn2) # components already validated so no quoting required
+    os.system("rm -r "+tempdir)
+
+def addWord(l1w,l2w,l1,l2,out=True):
+    if out: dirID=setup_userID()
+    if not (gradint.firstLanguage,gradint.secondLanguage) == (l1,l2):
+      if not ((gradint.firstLanguage,gradint.secondLanguage) == (l2,l1) and "HTTP_REFERER" in os.environ and not cginame in os.environ["HTTP_REFERER"]): gradint.updateSettingsFile(gradint.settingsFile,{"firstLanguage": l1,"secondLanguage":l2})
+      gradint.firstLanguage,gradint.secondLanguage = l1,l2
+    if (l1w+"_"+l1,l2w+"_"+l2) in map(lambda x:x[1:],gradint.parseSynthVocab(gradint.vocabFile,forGUI=1)):
+      if out: htmlOut(withLocalise('This word is already in your list.')+backLink)
+      return
+    gradint.appendVocabFileInRightLanguages().write(gradint.B(l2w)+gradint.B("=")+gradint.B(l1w)+gradint.B("\n"))
+    if not out: return
+    if "HTTP_REFERER" in os.environ and not cginame in os.environ["HTTP_REFERER"]: extra="&dictionary="+quote(os.environ["HTTP_REFERER"])
+    else: extra=""
+    redirectHomeKeepCookie(dirID,extra)
+
+def redirectHomeKeepCookie(dirID,extra=""):
+    dirID = gradint.S(dirID) # just in case
+    print ("Location: "+cginame+"?random="+str(random.random())[2:]+"&id="+dirID[dirID.rindex("/")+1:]+extra+"\n")
+
+def langSelect(name,curLang):
+    curLang = gradint.espeak_language_aliases.get(curLang,curLang)
+    return '<select name="'+name+'">'+''.join(['<option value="'+abbr+'"'+gradint.cond(abbr==curLang," selected","")+'>'+localise(abbr)+' ('+abbr+')'+'</option>' for abbr in sorted(langFullName.keys())])+'</select>'
+
+def numSelect(name,nums,curNum): return '<select name="'+name+'">'+''.join(['<option value="'+str(num)+'"'+gradint.cond(num==curNum," selected","")+'>'+str(num)+'</option>' for num in nums])+'</select>'
+
+def localise(x,span=0):
+    r=gradint.localise(x)
+    if r==x: return langFullName.get(gradint.espeak_language_aliases.get(x,x),x)
+    if span==1: r="<span lang=\""+gradint.firstLanguage+"\">"+r+"</span>"
+    elif span==2: r+='" lang="'+gradint.firstLanguage
+    if type(r)==type("")==type(u""): return r # Python 3
+    else: return r.encode('utf-8') # Python 2
+for k,v in {"Swap":{"zh":u"交换","zh2":u"交換"},
+            "Text edit":{"zh":u"文本编辑"},
+            "Delete":{"zh":u"删除","zh2":u"刪除"},
+            "Really delete this word?":{"zh":u"真的删除这个词?","zh2":u"真的刪除這個詞?"},
+            "Your word list":{"zh":u"你的词汇表","zh2":u"你的詞彙表"},
+            "click for audio":{"zh":u"击某词就听声音","zh2":u"擊某詞就聽聲音"},
+            "Repeats":{"zh":u"重复计数","zh2":u"重複計數"},
+            "To edit this list on another computer, type":{"zh":u"要是想在其他的电脑或手机编辑这个词汇表，请在别的设备打","zh2":u"要是想在其他的電腦或手機編輯這個詞彙表，請在別的設備打"},
+            "Please wait while the lesson starts to play":{"zh":u"稍等本课正开始播放","zh2":u"稍等本課正開始播放"},
+            "Bigger":{"zh":u"大"},"Smaller":{"zh":u"小"},
+            'You must type a word in the box before pressing the Speak button.':{"zh":u"按‘发音’前，应该框里打字。","zh2":u"按‘發音’前，應該框裡打字。"},
+            'You must type words in both boxes before pressing the Add button.':{"zh":u"按‘添加’前，应该在两框里打字。","zh2":u"按‘添加’前，應該在兩框裡打字。"},
+            'You must change the settings before pressing the Change Languages button.':{"zh":u"按‘选择其他语言’前，应该转换语言设定。","zh2":u"按‘選擇其他語言’前，應該轉換語言設定。"},
+            'This word is already in your list.':{"zh":u"本词已经在您的词汇表。","zh2":u"本詞已經在您的詞彙表。"},
+            "Your word list is empty.":{"zh":u"词汇表没有词汇，加一些吧","zh2":u"詞彙表沒有詞彙，加一些吧"}
+            }.items():
+  if not k in gradint.GUI_translations: gradint.GUI_translations[k]=v
+
+def withLocalise(x): return x+"  "+localise(x,1)
+
+def h5a():
+    body = """<script><!--
+function h5a(link,endFunc) { if (document.createElement) {
+   var ae = document.createElement('audio');
+   function cp(t,lAdd) { if(ae.canPlayType && function(s){return s!="" && s!="no"}(ae.canPlayType(t))) {
+       if (link.href) ae.setAttribute('src', link.href+lAdd);
+       else ae.setAttribute('src', link+lAdd);
+       if (typeof endFunc !== 'undefined') { ae.addEventListener("ended", endFunc, false); ae.addEventListener("timeupdate",function(e){t=ae.currentTime;m=Math.floor(t/60);t=Math.floor(t%60);document.forms[0].lesson.value=m+(t<10?":0":":")+t},false) }
+       ae.play(); return true; // Safari can say "Unhandled Promise Rejection: AbortError: The operation was aborted." in console log, but plays anyway when loaded
+    } return false; }
+   if (cp('audio/mpeg','')) return false;"""
+    if gradint.got_program("oggenc"): body += """else if (cp('audio/ogg',"&filetype=ogg")) return false;"""
+    body += """} return true; }
+//--></script>"""
+    return body
+def hasVoiceOptions(l):
+    if not l in gradint.synth_partials_voices: return False
+    if not gradint.guiVoiceOptions: return False
+    try: voices = os.listdir(gradint.partialsDirectory+os.sep+l)
+    except: voices = []
+    for v in voices:
+        if "-" in v and v[:v.index("-")] in voices: return True
+def listVocab(hasList): # main screen
+    firstLanguage,secondLanguage = gradint.firstLanguage, gradint.secondLanguage
+    # TODO button onClick: careful of zh w/out tones, wld need to JS this
+    body = h5a() + '<center><form action="'+cginame+'">'
+    body += '<input type=submit name=bigger value="%s"> | <input type=submit name=smaller value="%s">' % (localise("Bigger"),localise("Smaller"))
+    GUIlang = gradint.GUI_languages.get(firstLanguage,firstLanguage)
+    if "@variants-"+GUIlang in gradint.GUI_translations:
+      body += " -- " # separating from big/small
+      count = 0
+      for variant in gradint.GUI_translations["@variants-"+GUIlang]:
+        if count: body += " | "
+        body += '<input type=submit name="variant'+str(count)+'" value="'+gradint.cond(type("")==type(u""),variant,variant.encode('utf-8'))+'"'+gradint.cond(gradint.scriptVariants.get(GUIlang,0)==count,' disabled="disabled"',"")+'>'
+        count += 1
+    body += "<br>"
+    gotVoiceOptions = (hasVoiceOptions(gradint.secondLanguage) or hasVoiceOptions(gradint.firstLanguage))
+    if gotVoiceOptions:
+      body += 'Voice option: <input type=submit name=voNormal value="Normal"'+gradint.cond(gradint.voiceOption=="",' disabled="disabled"',"")+'>'
+      for v in gradint.guiVoiceOptions: body += ' | <input type=submit name=vopt value="'+v[1].upper()+v[2:]+'"'+gradint.cond(gradint.voiceOption==v,' disabled="disabled"',"")+'>'
+      body += '<input type=hidden name=curVopt value="'+gradint.voiceOption+'">' # ignored by gradint.cgi but needed by browser cache to ensuer 'change voice option and press Speak again' works
+      body += '<br>'
+    # must have autocomplete=off if capturing keycode 13
+    if gotVoiceOptions: cacheInfo="&curVopt="+gradint.voiceOption
+    else: cacheInfo=""
+    body += (localise("Word in %s",1) % localise(secondLanguage))+': <input type=text name=l2w autocomplete=off onkeydown="if(event.keyCode==13) {document.forms[0].spk.click();return false} else return true" onfocus="document.forms[0].onsubmit=\'document.forms[0].onsubmit=&quot;return true&quot;;document.forms[0].spk.click();return false\'" onblur="document.forms[0].onsubmit=\'return true\'"> <input type=submit name=spk value="'+localise("Speak",2)+'" onClick="if (!document.forms[0].l1w.value && !document.forms[0].l2w.value) return true; else return h5a(\''+cginame+'?spk=1&l1w=\'+document.forms[0].l1w.value+\'&l2w=\'+document.forms[0].l2w.value+\'&l1=\'+document.forms[0].l1.value+\'&l2=\'+document.forms[0].l2.value+\''+cacheInfo+'\');"><br>'+(localise("Meaning in %s",1) % localise(firstLanguage))+': <input type=text name=l1w autocomplete=off onkeydown="if(event.keyCode==13) {document.forms[0].add.click();return false} else return true" onfocus="document.forms[0].onsubmit=\'document.forms[0].onsubmit=&quot;return true&quot;;document.forms[0].add.click();return false\'" onblur="document.forms[0].onsubmit=\'return true\'"> <input type=submit name=add value="'+(localise("Add to %s",2) % localise("vocab.txt").replace(".txt",""))+'"><script><!--\nvar emptyString="";document.write(\' <input type=submit name=placeholder value="'+localise("Clear input boxes",2)+'" onClick="document.forms[0].l1w.value=document.forms[0].l2w.value=emptyString;document.forms[0].l2w.focus();return false">\')\n//--></script><p><nobr>'+localise("Your first language",1)+': '+langSelect('l1',firstLanguage)+'</nobr> <nobr>'+localise("second",1)+': '+langSelect('l2',secondLanguage)+'</nobr> <nobr><input type=submit name=clang value="'+localise("Change languages",2)+'"><input type=submit name=swaplang value="'+localise("Swap",2)+'"></nobr>' # onfocus..onblur updating onsubmit is needed for iOS "Go" button
+    def htmlize(l,lang):
+       if type(l)==type([]) or type(l)==type(()): return htmlize(l[-1],lang)
+       l = gradint.B(l)
+       if gradint.B("!synth:") in l: return htmlize(l[l.index(gradint.B("!synth:"))+7:l.rfind(gradint.B("_"))],lang)
+       return justsynthLink(l,lang)
+    def deleteLink(l1,l2):
+       r = []
+       for l in [l2,l1]:
+         if type(l)==type([]) or type(l)==type(()) or not gradint.B("!synth:") in gradint.B(l): return "" # Web-GUI delete in poetry etc not yet supported
+         l = gradint.B(l)
+         r.append(gradint.S(quote(l[l.index(gradint.B("!synth:"))+7:l.rfind(gradint.B("_"))])))
+       r.append(localise("Delete",2))
+       return ('<td><input type=submit name="del-%s%%3d%s" value="%s" onClick="return confirm(\''+localise("Really delete this word?")+'\');"></td>') % tuple(r)
+    if hasList:
+       gradint.availablePrompts = gradint.AvailablePrompts() # needed before ProgressDatabase()
+       # gradint.cache_maintenance_mode=1 # don't transliterate on scan -> NO, including this scans promptsDirectory!
+       gradint.ESpeakSynth.update_translit_cache=lambda *args:0 # do it this way instead
+       data = gradint.ProgressDatabase().data ; data.reverse()
+       if data: hasList = "<p><table style=\"border: thin solid green\"><caption><nobr>"+localise("Your word list",1)+"</nobr> <nobr>("+localise("click for audio",1)+")</nobr> <input type=submit name=edit value=\""+localise("Text edit",2)+"\"></caption><tr><th>"+localise("Repeats",1)+"</th><th>"+localise(gradint.secondLanguage,1)+"</th><th>"+localise(gradint.firstLanguage,1)+"</th></tr>"+"".join(["<tr class=\"notranslate\"><td>%d</td><td lang=\"%s\">%s</td><td lang=\"%s\">%s</td>%s" % (num,gradint.secondLanguage,htmlize(dest,gradint.secondLanguage),gradint.firstLanguage,htmlize(src,gradint.firstLanguage),deleteLink(src,dest)) for num,src,dest in data])+"</table>"
+       else: hasList=""
+    else: hasList=""
+    if hasList: body += '<p><table style="border:thin solid blue"><tr><td>'+numSelect('new',range(2,10),gradint.maxNewWords)+' '+localise("new words in")+' '+numSelect('mins',[15,20,25,30],int(gradint.maxLenOfLesson/60))+' '+localise('mins')+""" <input type=submit name=lesson value="""+'"'+localise("Start lesson",2)+"""" onClick="document.forms[0].lesson.disabled=1; document.forms[0].lesson.value=&quot;"""+localise("Please wait while the lesson starts to play")+"""&quot;;document.d0=new Date();return h5a('"""+cginame+'?lesson='+str(random.random())[2:]+"""&h5a=1&new='+document.forms[0].new.value+'&mins='+document.forms[0].mins.value,function(){if(new Date()-document.d0>60000)location.href='"""+cginame+'?lFinish='+str(random.random())[2:]+"""';else{document.forms[0].lesson.value='PLAY ERROR'}})"></td></tr></table>""" # when lesson ended, refresh with lFinish which saves progress (interrupts before then cancel it), but don't save progress if somehow got the ended event in 1st minute as that could be a browser issue
+    if "dictionary" in query:
+        if query.getfirst("dictionary")=="1": body += '<script><!--\ndocument.write(\'<p><a href="javascript:history.go(-1)">'+localise("Back to referring site",1)+'</a>\')\n//--></script>' # apparently it is -1, not -2; the redirect doesn't count as one (TODO are there any JS browsers that do count it as 2?)
+        else: body += '<p><a href="'+query.getfirst("dictionary")+'">'+localise("Back to dictionary",1)+'</a>' # TODO check for cross-site scripting
+    if hasList:
+      if "://" in gradintUrl: hasList += "<p>"+localise("To edit this list on another computer, type",1)+" <kbd>"+gradintUrl.replace(".","<wbr>.").replace("/","<wbr>/")+"?id="+re.sub("([0-9]{4})(?!$)",r"\1<wbr><span><!-- (this is not a phone number) --></span>",getCookieId())+"</kbd>" # span needed for iOS at least
+    else: hasList="<p>"+localise("Your word list is empty.",1)
+    body += hasList
+    htmlOut(body+'</form></center><script><!--\ndocument.forms[0].l2w.focus()\n//--></script>')
+
+def has_userID(): # TODO: can just call getCookieId with not too much extra overhead
+    cookie_string = os.environ.get('HTTP_COOKIE',"")
+    if cookie_string:
+        cookie = Cookie.SimpleCookie()
+        cookie.load(cookie_string)
+        return 'id' in cookie
+
+def getCookieId():
+    cookie_string = os.environ.get('HTTP_COOKIE',"")
+    if not cookie_string: return
+    cookie = Cookie.SimpleCookie()
+    cookie.load(cookie_string)
+    if 'id' in cookie: return cookie['id'].value.replace('"','').replace("'","").replace("\\","")
+
+def setup_userID():
+    # MUST call before outputting headers (may set cookie)
+    # Use the return value of this with -settings.txt, -vocab.txt etc
+    if cginame=="gradint.cgi": dirName = "cgi-gradint-users" # as previous versions
+    else: dirName = cginame+"-users" # TODO document this feature (you can symlink something-else.cgi to gradint.cgi and it will have a separate user directory) (however it still reports gradint.cgi on the footer)
+    if not os.path.exists(dirName): os.system("mkdir "+dirName)
+    userID = getCookieId()
+    need_write = (userID and not os.path.exists(dirName+'/'+userID+'-settings.txt')) # maybe it got cleaned up
+    if not userID:
+        while True:
+            userID = str(random.random())[2:]
+            if not os.path.exists(dirName+'/'+userID+'-settings.txt'): break
+        open(dirName+'/'+userID+'-settings.txt','w') # TODO this could still be a race condition (but should be OK under normal circumstances)
+        need_write = 1
+        print ('Set-Cookie: id=' + userID+'; expires=Wed, 1 Dec 2036 23:59:59 GMT') # TODO: S2G
+    userID0, userID = userID, dirName+os.sep+userID # already validated
+    gradint.progressFileBackup=gradint.pickledProgressFile=None
+    gradint.vocabFile = userID+"-vocab.txt"
+    gradint.progressFile = userID+"-progress.txt"
+    gradint.settingsFile = userID+"-settings.txt"
+    gradint.outputPrefix = myTmp+os.sep+userID0+"-"
+    if need_write: gradint.updateSettingsFile(gradint.settingsFile,{'firstLanguage':gradint.firstLanguage,'secondLanguage':gradint.secondLanguage})
+    else: gradint.readSettings(gradint.settingsFile)
+    gradint.auto_advancedPrompt=1 # prompt in L2 if we don't have L1 prompts on the server, what else can we do...
+    if os.path.exists(userID+"-zoom.txt"):
+      global zoom ; zoom = int(open(userID+"-zoom.txt").read().strip())
+    return userID
+
+try: main()
+except Exception as e:
+  print ("Content-type: text/plain; charset=utf-8\n")
+  sys.stdout.flush()
+  import traceback
+  try: traceback.print_exc(file=sys.stdout)
+  except: pass
+  sys.stdout.flush()
+  if hasattr(sys.stdout,"buffer"): buf = sys.stdout.buffer
+  else: buf = sys.stdout
+  buf.write(repr(e).encode("utf-8"))

server/lesson-table.py

+#!/usr/bin/env python
+# (compatible with both Python 2 and Python 3)
+
+# Script to generate an HTML table of the contents of a lesson
+# for summarizing it to a teacher or native speaker.
+# Reads from progressFile and progressFileBackup.
+
+# Version 1.07 (c) 2011, 2020-21, 2025 Silas S. Brown.  License: GPL
+
+# Example use:
+# export samples_url=http://example.org/path/to/samples/ # or omit
+# python lesson-table.py [gradint-params] | ssh some-server 'mutt -e "set record = \"\";" -e "set charset=\"utf-8\"; set send_charset=\"utf-8\"; set content_type=\"text/html\";" to-address -s "Gradint report"' || echo Send failed
+
+import gradint, os
+samples_url = os.getenv("samples_url","")
+
+from gradint import B,S
+newpf = gradint.progressFile
+gradint.progressFile = gradint.progressFileBackup
+gradint.pickledProgressFile=None
+
+mergeIn = gradint.scanSamples()+gradint.parseSynthVocab(gradint.vocabFile)
+
+oldProg = gradint.ProgressDatabase(alsoScan=0)
+oldProg.data += oldProg.unavail # because it might be available in newProg
+gradint.mergeProgress(oldProg.data,mergeIn)
+opd = {}
+for tries,l1,l2 in oldProg.data:
+  key = gradint.norm_filelist(l1,l2)
+  if tries: opd[key]=tries
+del oldProg
+gradint.progressFile = newpf
+newProg = gradint.ProgressDatabase(alsoScan=0)
+gradint.mergeProgress(newProg.data,mergeIn)
+del mergeIn
+changes = [] ; count=0
+gradint.sort(newProg.data,gradint.cmpfunc)
+for tries,l1,l2 in newProg.data:
+  if not tries: continue
+  key = gradint.norm_filelist(l1,l2)
+  oldTries = opd.get(key,0)
+  if not oldTries==tries: changes.append((oldTries,count,tries-oldTries,S(l1),S(l2)))
+  count += 1
+del newProg,opd
+changes.sort()
+print ('<html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"><title>Gradint lesson report</title><meta name="mobileoptimized" content="0"><meta name="viewport" content="width=device-width"><script>if(window.matchMedia && window.matchMedia("(prefers-color-scheme: dark)").matches)document.write("<style>body { background-color: black; color: #c0c000; } a:link { color: #00b000; } a:visited { color: #00c0c0; } a:hover { color: red; }</style>");if(navigator.languages && navigator.languages.indexOf("en")>=0) document.write(\'<meta name="google" content="notranslate">\')</script></head><body><h2>Gradint lesson report</h2>')
+if gradint.unix and gradint.got_program("zgrep"):
+  print (os.popen("zgrep '^# collection=' \"%s\"" % gradint.progressFile).read()[2:].rstrip())
+print ('<table border><tr><th>Repeats before</th><th>Repeats today</th><th>Question</th><th>Answer</th></tr>') # (have Question/Answer order rather than Word/Meaning, because if it's L2-only poetry then the question is the previous line, which is not exactly "meaning")
+  
+had_h5a = False
+def h5aCode(filename):
+  r = real_h5aCode(filename)
+  if r:
+    global had_h5a
+    if not had_h5a:
+      had_h5a = True
+      print ("""<script language="Javascript"><!--
+function h5a(link,type) { if (document.createElement) {
+   var ae = document.createElement('audio');
+   if (ae.canPlayType && function(s){return s!="" && s!="no"}(ae.canPlayType(type))) {
+     if (link.href) ae.setAttribute('src', link.href);
+     else ae.setAttribute('src', link);
+     ae.play();
+     return false; } } return true; }
+//--></script>""")
+  return r
+def real_h5aCode(filename):
+  if filename.endswith(gradint.dotmp3): return ' onClick="javascript:return h5a(this,\'audio/mpeg\')"'
+  elif filename.endswith(gradint.dotwav): return ' onClick="javascript:return h5a(this,\'audio/wav\')"'
+  else: return ""
+
+def wrappable(f):
+  z = u'\u200b' # zero-width space
+  if not type(u"")==type(""): z=z.encode('utf-8') # Py2
+  return f.replace(os.sep,os.sep+z).replace('_',z+'_')
+def checkVariant(l,ensureTxt=0):
+  l=S(l)
+  if os.sep in l: fname=l[l.rindex(os.sep)+1:]
+  else: fname=l
+  variants = map(S,gradint.variantFiles.get(B(gradint.samplesDirectory+os.sep+l),[fname]))
+  if fname in variants: return l # ok
+  # else no default variant, need to pick one for the link
+  for v in variants:
+    if ensureTxt:
+      if not v.endswith(gradint.dottxt): continue
+    elif v.endswith(gradint.dottxt): continue
+    if not os.sep in l: return v
+    return l[:l.rindex(os.sep)+1]+v
+def link(l):
+  if type(l)==type([]): return link(l[-1])
+  l = S(l)
+  if l.lower().endswith(gradint.dottxt): l="!synth:"+S(gradint.u8strip(gradint.read(gradint.samplesDirectory+os.sep+checkVariant(l,1)))).strip(gradint.wsp)+"_"+gradint.languageof(l)
+  if "!synth:" in l:
+    if gradint.languageof(l) not in [gradint.firstLanguage,gradint.secondLanguage]: l=S(gradint.textof(l))+" ("+gradint.languageof(l)+")"
+    else: l=S(gradint.textof(l))
+    return l.replace('&','&amp;').replace('<','&lt;')
+  if samples_url: return '<A HREF="'+samples_url+checkVariant(l)+'"'+h5aCode(checkVariant(l))+'>'+wrappable(l)+'</A>'
+  return wrappable(l).replace('&','&amp;').replace('<','&lt;')
+for b4,pos,today,l1,l2 in changes: print ('<tr><td>%d</td><td>%d</td><td class="notranslate">%s</td><td class="notranslate">%s</td></tr>' % (b4,today,link(l1),link(l2)))
+print ('</table></body></html>')

gradint-build/samples/utils/safety-check-progressfile.py → server/safety-check-progressfile.py

+#!/usr/bin/env python
+# (either Python 2 or Python 3)
+
 # safety-check-progressfile.py:
 # The purpose of this script is to check
 # progress.txt for safety.  Because it's

server/samples.cgi

+#!/bin/bash
+
+# Gradint online samples browser v1.1 (c) 2011,2013 Silas S. Brown.  License: GPL
+
+# Works as an "indexing" CGI.
+# To set up in Apache, make .htaccess with:
+# Options -Indexes
+# ErrorDocument 403 /~your-user-ID/cgi-bin/samples.cgi
+# <FilesMatch "\.(txt)$">
+# ForceType 'text/plain; charset=UTF-8' 
+# </FilesMatch>
+
+# and change the /home/ssb22 in the script below.
+
+# To set up in mathopd, configure like this:
+#		Control {
+#		  Alias /samples
+#		  Location /home/userID/gradint/samples/
+#		  AutoIndexCommand /home/userID/path/to/samples.cgi
+#		}
+
+# and delete the REQUEST_URI logic below.
+
+# You can override this script in selected (sub)directories
+# by making index.html files for those.
+
+if ! test "a$REQUEST_URI" == a; then
+  cd "/home/ssb22/public_html/$(echo "$REQUEST_URI"|sed -e 's/?.*//')"
+fi # else assume the server put us in the right directory, like mathopd does
+
+if echo "$SERVER_SOFTWARE"|grep Apache >/dev/null; then
+  echo "Status: 200 OK" # overriding the 403
+fi # (mathopd doesn't need this, and not tested with all mathopd versions)
+
+Filename="$(pwd|sed -e 's,.*/,,').zip"
+
+if test "$QUERY_STRING" == zip || test "a$(echo "$REQUEST_URI"|sed -e 's/.*?//')" == azip; then
+  echo Content-type: application/zip
+  echo "Content-Disposition: attachment; filename=$Filename"
+  echo
+  cd .. ; zip -9r - "$(echo "$Filename"|sed -e s/.zip$//)"
+else
+  echo "Content-type: text/html; charset=utf-8"
+  echo
+  echo "<HTML><BODY><A HREF=\"..\">Parent directory</A> |"
+  echo "<A HREF=\"./?zip\">Download $Filename</A> (expands to $(du -h --apparent-size -s|cut -f1))"
+  echo "<h2>Contents of $Filename</h2><UL>"
+  cat <<EOF
+<script language="Javascript"><!--
+function h5a(link) {
+ if (!link.nextSibling) return true;
+ if (link.nextSibling.src) {
+   link.nextSibling.play();
+   return false;
+ } else {
+   var ae = document.createElement('audio');
+   var atype;
+   if (link.href.match("mp3$")) atype="audio/mpeg";
+   else if (link.href.match("wav$")) atype="audio/wav";
+   else if (link.href.match("ogg$")) atype="audio/ogg";
+   else return true;
+   if (ae.canPlayType && function(s){return s!="" && s!="no"}(ae.canPlayType(atype))) {
+ ae.setAttribute('src', link.href);
+ ae.setAttribute('controls', 'controls');
+ link.parentNode.insertBefore(ae,link.nextSibling);
+ ae.play();
+ return false;
+   }
+} return true;}
+//--></script>
+EOF
+  for N in *; do
+    Size=$(du -h --apparent-size -s "$N"|cut -f1)
+    if echo "$N"|grep '\.txt$'>/dev/null && echo $Size|grep '^[0-9]*$' >/dev/null;then Size="$(cat "$N")";else Size="($Size)"; fi
+    echo "<LI><A HREF=\"$N\" onClick=\"javascript:return h5a(this)\">$N</A> $Size</LI>"
+  done
+  echo "</UL></BODY></HTML>"
+fi

gradint-build/samples/utils/vocab2html.py → server/vocab2html.py

@@ -13,34 +13,35 @@
 # be used.  E.g.: export ESPEAK_CGI_URL="/~userID/espeak.cgi"
 # (TODO: this script ignores the possibility of synthesizing phrases from partials)
 
-# (c) Silas S. Brown, License: GPL
+# Version 1.2, (c) Silas S. Brown, License: GPL
 
 from gradint import *
 if not synthCache: synthCache_contents = []
 langs=[secondLanguage,firstLanguage]
 o=open(vocabFile,"rU")
 justHadP=1
-sys.stdout.write("<HEAD><META HTTP_EQUIV=Content-type CONTENT=\"text/html; charset=utf-8\"></HEAD>\n") # (assume utf8 in case there's any hanzi in lily, or in espeak cantonese voice or whatever - but TODO what if using another charset for another language?)
-for l in o.readlines():
+print ('<html><HEAD><META HTTP-EQUIV=Content-type CONTENT="text/html; charset=utf-8"><meta name="viewport" content="width=device-width"></HEAD><body>') # (assume utf8 in case there's any hanzi, but TODO what if using another charset for another language?)
+for l in o:
   l2=l.lower()
   if l2.startswith("set language ") or l2.startswith("set languages "): langs=l.split()[2:]
   if not l.strip():
     # blank line
-    if not justHadP: sys.stdout.write("<P>")
+    if not justHadP: print ("<P>")
     justHadP=1 ; continue
-  if not justHadP: sys.stdout.write("<BR>")
+  if not justHadP: print ("<BR>")
   if l2.startswith("set language ") or l2.startswith("set languages ") or l2.startswith("limit on") or l2.startswith("limit off") or l2.startswith("begin poetry") or l2.startswith("end poetry"):
-    sys.stdout.write("<EM>%s</EM>" % (l,))
+    print ("<EM>%s</EM>" % (l,))
   elif l2.startswith("#"):
     # comment (and may be part of multi-line comment)
-    if not l[1:].strip().startswith("<!--"): sys.stdout.write("<small>#</small> ")
-    sys.stdout.write(l[1:])
+    if not l[1:].strip().startswith("<!--"): print ("<small>#</small> ")
+    print (l[1:])
   else:
     # vocab line
     langsAndWords=zip(langs,map(lambda x:x.strip(),l.split("=")))
     out = []
     for lang,word in langsAndWords:
-      fname=synthCache_transtbl.get(word.lower()+"_"+lang+dotwav,word.lower()+"_"+lang+dotwav)
+      lang,word = S(lang),S(word)
+      fname=S(synthCache_transtbl.get(word.lower()+"_"+lang+dotwav,word.lower()+"_"+lang+dotwav))
       found = 0
       for fn2 in [fname,fname.replace(dotwav,dotmp3)]:
           if fn2 in synthCache_contents:
@@ -48,8 +49,10 @@ for l in o.readlines():
               found = 1 ; break
       if not found:
           if os.getenv("ESPEAK_CGI_URL"):
-              import urllib
-              out.append("<A HREF=\""+os.getenv("ESPEAK_CGI_URL")+"?"+urllib.urlencode({"t":word,"l":lang})+"\">"+word+"</A>")
+              try: from urllib import urlencode # Python 2
+              except: from urllib.parse import urlencode # Python 3
+              out.append("<A HREF=\""+os.getenv("ESPEAK_CGI_URL")+"?"+urlencode({"t":word,"l":lang})+"\">"+word+"</A>")
           else: out.append(word)
-    sys.stdout.write(" = ".join(out))
+    print (" = ".join(out))
   justHadP=0
+print ("</body></html>")