gradint-build/samples/utils/README.txt → samples/utils/README.txt

@@ -10,6 +10,8 @@ C:\Windows\system32 or somewhere.
 
 All require Python (from www.python.org).  All systems
 except Windows have that anyway if you can run gradint.
+However, some of these utilities are still Python2-only,
+while the main Gradint is now compatible with both 2 and 3.
 
 SPLITTING SOUND FILES
 ---------------------
@@ -18,13 +20,10 @@ autosplit.py - splits a long recording into individual files
 completely automatically, but only if the recording has been
 made in near-broadcasting-studio conditions.
 
-strip0.py - strips absolute silence off the beginning and
-end of audio files (only useful if you're dealing with files
-from a textbook CD or something)
-
 manual-splitter.py (Unix only but not too difficult to
 modify for other systems) - a helper script so that you can
-use Audacity (or another sound editor) to split the file.
+use Audacity (or another sound editor) to split the file
+in non-realtime.  (Realtime splitting can be done in Gradint.)
 Use the "export selection as wav" command (you can assign a
 hot-key to it), and you don't have to type in a different
 filename each time because this script can run in the
@@ -51,4 +50,8 @@ online synthesizers (or real people) to the synth cache
 transliterate.py - make a transliterated vocab report
 (for use with grep or on PDAs or whatever)
 
+player.py - play 
+
 diagram.py - make a diagram of a gradint lesson
+
+trace.py - make a raytraced animation of a lesson

gradint-build/samples/utils/autosplit.py → samples/utils/autosplit.py

@@ -4,13 +4,12 @@ import os, struct, sndhdr, sys
 try: import winsound
 except: winsound=None
 macsound = (sys.platform.find("mac")>=0 or sys.platform.find("darwin")>=0)
-if macsound: sys.stderr.write("Warning: You need to have qtplay (from gradint or wherever) in your PATH for this to work\n")
 
 # python 3+:
 try: input
 except: input=lambda x:eval(raw_input(x))
 
-startCount = 1   # or 0, or 485 or whatever
+startCount = 0   # or however many WAVs already exist (should be even)
 
 threshold = 10 # 3 is too low for recorded sound, but if using speech synth you might want to set it to 1
 shortestSilence = 0.3
@@ -18,17 +17,29 @@ shortestSound = 0.4
 
 if len(sys.argv)>1: exec(" ".join(sys.argv[1:])) # so you can override the above on the command line
 
+sox_8bit, sox_16bit, sox_32bit, sox_signed, sox_unsigned = "-b", "-w", "-l", "-s", "-u"
+if not winsound: # adapted from gradint (see comments there)
+  sox_formats=os.popen("sox --help 2>&1").read()
+  sf2 = ' '.join(sox_formats.lower().split())
+  if sf2.startswith("sox: sox v"):
+    if sf2[10]==' ': soxMaj=15
+    else: soxMaj = int(sf2[10:sf2.index('.')])
+  else: soxMaj=0
+  if soxMaj>=14:
+    if soxMaj==14 and sf2[13]<'4': sox_8bit, sox_16bit, sox_32bit = "-1", "-2", "-4"
+    else: sox_8bit, sox_16bit, sox_32bit, sox_signed, sox_unsigned = "-b 8", "-b 16", "-b 32", "-e signed-integer", "-e unsigned-integer" # TODO: check the last one
+
 def autosplit(filename,lang1,lang2,threshold):
     (wtype,rate,channels,wframes,bits) = sndhdr.what(filename)
     if bits==8:
-        soxBits="-b -u"
+        soxBits=sox_8bit+" "+sox_unsigned
         structBits="B"
     elif bits==16:
-        soxBits="-w -s"
+        soxBits=sox_16bit+" "+sox_signed
         structBits="h"
         threshold *= 256
     elif bits==32:
-        soxBits="-l -s"
+        soxBits=sox_32bit+" "+sox_signed
         structBits="i"
         threshold *= (256 * 256 * 256)
     else: raise Exception("Unsupported bits per sample")
@@ -47,7 +58,8 @@ def autosplit(filename,lang1,lang2,threshold):
         (sounding, bytes) = nextSample()
         if inSilence and not sounding and bytes: continue
         elif bytes:
-            dataToWriteout.append(bytes)
+            if dataToWriteout or sounding:
+                dataToWriteout.append(bytes)
             if sounding: numSilences = inSilence = 0
             else: numSilences += 1
         if numSilences >= int(shortestSilence*rate) or not bytes:
@@ -64,8 +76,9 @@ def autosplit(filename,lang1,lang2,threshold):
                 open(fname, "wb").write(''.join(dataToWriteout))
                 os.system("sox %s \"%s\" \"%s.wav\"" % (soxParams,fname,fname))
                 os.unlink(fname)
+                print fname+".wav"
                 if winsound: winsound.PlaySound(fname+".wav",winsound.SND_FILENAME)
-                elif macsound: os.system("qtplay "+fname+".wav")
+                elif macsound: os.system("afplay "+fname+".wav")
                 else: os.system("play "+fname+".wav")
             # Anyway, clear the output buffer
             dataToWriteout = []

gradint-build/samples/utils/cache-synth.py → samples/utils/cache-synth.py

-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 # cache-synth.py [--test] language [language ...]
 

gradint-build/samples/utils/cleanup-cache.py → samples/utils/cleanup-cache.py

-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 # delete cached synthesized words that are not used
 # (i.e. not mentioned in vocab.txt or samples).

gradint-build/samples/utils/diagram.py → samples/utils/diagram.py

+#!/usr/bin/env python2
 
 # diagram.py: script to generate diagrams of gradint lessons
 # (C) 2008 Silas S. Brown.  License: GPL
 
-# gradint is run normally (passing any command-line arguments on)
+# Gradint is run normally (passing any command-line arguments on)
 # and then a diagram of the lesson it made is written to diagram.svg
 
 # you can get .ps by doing: inkscape -p '> diagram.ps' diagram.svg
@@ -59,7 +60,7 @@ def CompositeEvent_draw(self,startTime,pixelsPerSec,topY,height):
 gradint.CompositeEvent.draw=CompositeEvent_draw
 
 def Event_colour(self,language):
-    if hasattr(self,"wordToCancel"):
+    if self.makesSenseToLog():
       if language==gradint.firstLanguage: return "yellow" # TODO: 2nd to 3rd lang etc?
       else: return "green"
     else: return "grey" # prompts

samples/utils/filemove.sh

+#!/bin/bash
+
+SamplesDir="samples/" # Must include trailing /
+ProgressFile="progress.txt"
+if ! [ -e $SamplesDir ]; then echo "Error: $SamplesDir does not exist (are you in the right directory?)"; exit 1; fi
+if ! [ -e $ProgressFile ]; then echo "Error: $ProgressFile does not exist (are you in the right directory?)";exit 1;fi
+
+if test "a$1" == a; then
+  echo "Usage: $0 oldname newname"
+  echo "oldname and newname are relative to $SamplesDir, and can be prefixes of several files/directories"
+  echo "Moves files from one samples directory to another, keeping $ProgressFile adjusted.  Make sure gradint is not running (including waiting for start) when in use."
+  exit 1
+fi
+
+Src=$1
+Dest=$2
+
+find "$SamplesDir" -follow -type f | grep "^$SamplesDir$Src" | \
+while true; do read || break;
+  SrcFile=$REPLY
+  DestFile=$(echo "$SrcFile"|sed -e "s|^$SamplesDir$Src|$SamplesDir$Dest|")
+  mkdir -p "$DestFile" ; rmdir "$DestFile" # ensure parent dirs exist before moving file across
+  mv -b "$SrcFile" "$DestFile"
+  SrcFile=$(echo "$SrcFile"|sed -e "s|$SamplesDir||")
+  DestFile=$(echo "$DestFile"|sed -e "s|$SamplesDir||")
+  gzip -fdc "$ProgressFile" | sed -e "s|$SrcFile|$DestFile|g" > /tmp/newprog ; mv /tmp/newprog "$ProgressFile" # (ideally should re-write to batch these changes, but leave like this for now in case need to recover from unfinished operation)
+done
+
+rmdir "$SamplesDir$Src" 2>/dev/null >/dev/null # IF it's a directory

gradint-build/samples/utils/list-synth.py → samples/utils/list-synth.py

-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 # list-synth.py language [language ...]
 # list all words that can be synthesized

gradint-build/samples/utils/list2cache.py → samples/utils/list2cache.py

-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 # list2cache.py language
 

gradint-build/samples/utils/manual-splitter.py → samples/utils/manual-splitter.py

 #!/usr/bin/env python
 
-# Like splitter.py, but lets you use Audacity etc to split in non-realtime.
+# Like Gradint's "record from file", but lets you use Audacity etc to split in non-realtime.
 # You must export the segments in order.
 
 # ('mv && increment count' in a loop: ok as long as in same dir so no complicatns w cross-device & still-open)

samples/utils/player.py

+#!/usr/bin/env python
+# (should work in both Python 2 and Python 3)
+
+# Simple sound-playing server v1.59
+# Silas S. Brown - public domain - no warranty
+
+# connect to port 8124 (assumes behind firewall)
+# and each connection can send WAV or MP3 data
+
+# so gradint advanced.txt can do
+# wavPlayer = mp3Player = "nc HostName 8124 -q 0 <"
+# (most of this script assumes GNU/Linux)
+
+import socket, select, os, sys, os.path, time, re
+for a in sys.argv[1:]:
+  if a.startswith("--rpi-bluetooth-setup"): # tested on Raspberry Pi 400 with OS versions 11 and 12; also tested on Raspberry Pi Zero W with Raspbian 10 Lite (with the device already paired: needed to say "scan on", "discovery on", remove + pair in bluetoothctl).  Send Eth=(bluetooth Ethernet addr) to start.  Note that the setup command reboots the system.
+    os.system(r'if ! grep "$(cat ~/.ssh/*.pub)" ~/.ssh/authorized_keys; then cat ~/.ssh/*.pub >> ~/.ssh/authorized_keys;fi && (echo "[Unit]";echo "Description=Gradint player utility";echo "[Service]";echo "Type=oneshot";echo "ExecStart=bash -c \"while ! ssh localhost true; do sleep 1; done; ssh localhost '+os.path.join(os.getcwd(),sys.argv[0])+r'\"";echo "WorkingDirectory='+os.getcwd()+'";echo User="$(whoami)";echo "[Install]";echo "WantedBy=multi-user.target") > player.service && sudo mv player.service /etc/systemd/system/ && sudo systemctl daemon-reload && sudo systemctl enable player && chmod +x '+sys.argv[0]+' && sudo bash -c "apt-get -y install sox mpg123 pulseaudio pulseaudio-module-bluetooth && usermod -G bluetooth -a $USER && (echo load-module module-switch-on-connect;echo load-module module-bluetooth-policy;echo load-module module-bluetooth-discover) >> /etc/pulse/default.pa && (echo [General];echo FastConnectable = true) >> /etc/bluetooth/main.conf && reboot"') # (eee off: improves reliability of gigabit ethernet on RPi400)
+  elif a=="--aplay": use_aplay = True # aplay and madplay, for older embedded devices, NOT tested together with --rpi-bluetooth-* above
+  elif a.startswith("--delegate="): delegate_to_check=a.split('=')[1] # will ping that IP and delegate all sound to it when it's up.  E.g. if it has better amplification but it's not always switched on.
+  elif a.startswith("--chime="): chime_mp3=a.split('=')[1] # if clock bell desired, e.g. echo '$i-14vfff$c48o0l1b- @'|mwr2ly > chime.ly && lilypond chime.ly && timidity -Ow chime.midi && audacity chime.wav (amplify + trim) + mp3-encode (keep default 44100 sample rate so ~38 frames per sec).  Not designed to work with --delegate.  Pi1's 3.5mm o/p doesn't sound very good with this bell.
+  else: assert 0, "unknown option "+a
+
+os.environ["PATH"] += ":/usr/local/bin"
+try: use_aplay
+except: use_aplay = False
+try: delegate_to_check
+except: delegate_to_check = None
+try: chime_mp3
+except: chime_mp3 = None
+last_chime = last_play = 0
+delegate_known_down = 0
+s=socket.socket()
+s.bind(('',8124))
+s.listen(5)
+if type(b"")==type(""): S=lambda x:x # Python 2
+else: S=lambda x:x.decode("latin1") # Python 3
+eth = ""
+while True:
+    if chime_mp3:
+        t = time.time()
+        if t > last_chime+60 and t%1800 < 60 and not t<last_play+20:
+            last_chime = t ; h,m=time.localtime(t)[3:5]
+            if m>1: numChimes = 1
+            elif not h%12: numChimes = 12
+            else: numChimes = h%12
+            if not 7<=h%24<=22: pass # silence the chime at night
+            elif use_aplay:
+              if numChimes > 1: os.system("(madplay -Q -t 1 -o wav:- '"+chime_mp3+"'"+(";madplay -Q -t 1 -o raw:- '"+chime_mp3+"'")*(numChimes-2)+";madplay -Q -o raw:- '"+chime_mp3+"') | aplay -q")
+              else: os.system("madplay -Q -o wav:- '%s' | aplay -q" % chime_mp3)
+            elif numChimes > 1: os.system("(mpg123 -w - -n 38 --loop %d '%s' ; mpg123 -s '%s') 2>/dev/null | play -t wav --ignore-length - 2>/dev/null" % (numChimes-1,chime_mp3,chime_mp3))
+            else: os.system("mpg123 -q '%s'" % chime_mp3)
+        if not select.select([s],[],[],1800-time.time()%1800)[0]: continue
+    c,(a,port) = s.accept()
+    c.settimeout(10)
+    try: d = S(c.recv(4))
+    except: # e.g. timeout, or there was an error reading the file on the remote side and we got 0 bytes
+        c.close() ; continue
+    if delegate_to_check and not a==delegate_to_check and delegate_known_down < time.time()-60 and not os.system("ping -c 1 -w 0.5 '"+delegate_to_check+"' >/dev/null 2>/dev/null"): player = "nc -N '"+delegate_to_check+"' 8124"
+    elif d=='RIFF': # WAV
+        if use_aplay: player = "aplay -q"
+        else: player = "play - 2>/dev/null"
+    elif d=='STOP':
+        c.close()
+        while not d=='START':
+            c,a = s.accept()
+            try: d = S(c.recv(5))
+            except: d = ""
+            c.close()
+        continue
+    elif d=='QUIT':
+        s.close() ; break
+    elif d=="Eth=": # Eth=ethernet address to connect via Bluetooth (see --rpi-bluetooth-setup above)
+        eth = S(c.recv(17))
+        assert re.match("^[A-Fa-f0-9:]+$",eth)
+        os.system("E="+eth+";if ! pacmd list-sinks | grep "+eth.replace(":","_")+" >/dev/null; then while true; do bluetoothctl --timeout 1 disconnect | grep Missing >/dev/null||sleep 5;T=5;while ! bluetoothctl --timeout $T connect $E | egrep \"Connection successful|Device $E Connected: yes\"; do sleep 5; T=10;bluetoothctl --timeout 1 devices;echo Retrying $E; done ; Got=0; for Try in 1 2 3 4 5 6 7 8 9 a b c d e f g h i j k l m n o p q r s t u v w x y z; do if pacmd list-sinks | grep "+eth.replace(":","_")+" >/dev/null; then Got=1; break; fi; sleep 1; done; if [ $Got = 1 ] ; then break; fi; done; fi; pacmd set-default-sink bluez_sink."+eth.replace(":","_")+".a2dp_sink") # ; play /usr/share/scratch/Media/Sounds/Animal/Dog1.wav # (not really necessary if using 'close the socket' to signal we're ready)
+        c.close() ; continue
+    elif d=="Eth0":
+      if eth: os.system("bluetoothctl --timeout 1 disconnect "+eth)
+      c.close() ; continue
+    elif use_aplay: player = "madplay -Q -o wav:- - | aplay -q" # MP3
+    else: player = "mpg123 - 2>/dev/null" # MP3 non-aplay
+    if delegate_known_down < time.time()-60 and not player.startswith("nc -N "): delegate_known_down = time.time()
+    player = os.popen(player,"w")
+    if type(d)==type(u""): d = d.encode("latin1")
+    while d:
+        try:
+            try: player.write(d)
+            except TypeError: # Python 3
+                player.buffer.write(d)
+        except IOError: break # it was probably killed
+        try: d = c.recv(4096)
+        except: d = ""
+    try:
+        c.close() ; player.close()
+    except: pass
+    last_play = time.time()

gradint-build/samples/utils/recover-unavail.py → samples/utils/recover-unavail.py

+#!/usr/bin/env python2
 
 # Script to recover vocabulary from the "unavailable"
 # entries in Gradint's progress file.  Use if for some

gradint-build/samples/utils/synth-batchconvert-helper.py → samples/utils/synth-batchconvert-helper.py

-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 # Script to assist with using TextAloud or similar program
 # that can batch-synthesize a collection of text files
@@ -8,7 +8,7 @@
 # words in vocab.txt etc, and rename the resulting *.mp3 or *.wav
 # files into the synth cache.
 
-# Should be useful if you are on Linux and want to run a
+# Should be useful if you are not on Windows and want to run a
 # non-English speech synth in the Windows Emulator (since
 # ptts can have trouble, but tools like TextAloud still work).
 # Note: This script currently assumes that the filesystem
@@ -43,9 +43,10 @@ delete_old = 1  # if 1 (and if sporadic) then older cached
 # this script moves them there, as that's how it identifies its
 # "own" mp3/wav files (as opposed to anything else you may have cached).
 
-actually_generate = 0 # if 1, will call gradint to actually
-# generate the cached sound using the default voice.  Might
-# be useful if you need to move it to another machine that
+actually_generate = 0 # if 1, will call gradint to generate
+# the cached sound using its choice of voice for that language,
+# instead of relying on your use of TextAloud etc.
+# Might be useful if you need to move it to another machine that
 # doesn't have that voice, and you still want to use sporadic
 # etc (like a more advanced version of cache-synth.py)
 testMode = 0 # if 1 and actually_generate is 1, will play too
@@ -79,17 +80,24 @@ for l in os.listdir(newStuff):
     if l.endswith(dottxt) and "_" in l:
         txt = open(newStuff+os.sep+l).read().decode('utf-16')
         txt = (sporadic+txt,l[l.rindex("_")+1:l.rindex(gradint.extsep)])
-        generating[txt] = 1 ; fname2txt[l[:l.rindex(gradint.extsep)]]=txt
+        generating[txt] = (None,l)
+        fname2txt[l[:l.rindex(gradint.extsep)]]=txt
 for l in os.listdir(newStuff):
     if l.endswith(dotwav) or l.endswith(dotmp3):
         k=l[:l.rindex(gradint.extsep)]
         if k in fname2txt: generating[fname2txt[k]]=newStuff+os.sep+l
-del fname2txt # now 'generating' maps (txt,lang) to 1 or filename
+del fname2txt # now 'generating' maps (txt,lang) to (None,txtFile) or filename
+for k,v in generating.items():
+    if type(v)==tuple and v[0]==None: # a previous run was interrupted
+        os.remove(newStuff+os.sep+v[1])
+        del generating[k]
 
 def getTxtLang(s):
     if '!synth:' in s and "_" in s: return gradint.textof(s).decode('utf-8'),gradint.languageof(s)
-    elif s.endswith(gradint.extsep+"txt"): return gradint.readText(s).decode('utf-8'), gradint.languageof(s)
-    else: return None,None
+    elif s.endswith(gradint.extsep+"txt"):
+        langToSynth = gradint.languageof(s)
+        if langToSynth==languageToCache: return gradint.readText(s).decode('utf-8'), langToSynth # else don't bother reading the file (it might be over ftpfs)
+    return None,None
 
 def decache(s):
     textToSynth,langToSynth = getTxtLang(s)
@@ -118,6 +126,11 @@ if sporadic:
 
 count = 0 ; toMove = []
 
+def rename(old,new):
+    # don't use os.rename - can get problems cross-device
+    open(new,"wb").write(open(old,"rb").read())
+    os.remove(old)
+
 def maybe_cache(s):
     textToSynth,langToSynth = getTxtLang(s)
     if not textToSynth: return
@@ -130,17 +143,15 @@ def maybe_cache(s):
     k = (textToSynth.lower(),langToSynth)
     if generating.has_key(k):
         if not generating[k]==1: # a file already exists
-            # don't use os.rename - can get problems cross-device
             fname = textToSynth.lower().encode('utf-8')+'_'+langToSynth+generating[k][generating[k].rindex(gradint.extsep):]
-            open(gradint.synthCache+os.sep+fname,"wb").write(open(generating[k],"rb").read())
+            rename(generating[k],gradint.synthCache+os.sep+fname)
             scld[fname] = 1
-            #open(gradint.synthCache+os.sep+textToSynth.lower().encode('utf-8')+'_'+langToSynth+dottxt,"wb").write(open(generating[k][:generating[k].rindex(gradint.extsep)]+dottxt,"rb").read())
-            os.remove(generating[k])
+            #rename(generating[k][:generating[k].rindex(gradint.extsep)]+dottxt,gradint.synthCache+os.sep+textToSynth.lower().encode('utf-8')+'_'+langToSynth+dottxt)
             os.remove(generating[k][:generating[k].rindex(gradint.extsep)]+dottxt)
             generating[k]=1
         return
     if actually_generate:
-        tm = [gradint.synth_event(langToSynth,textToSynth.encode('utf-8')).getSound(),(textToSynth.encode('utf-8')+"_"+langToSynth+dotwav).lower()]
+        tm = [gradint.synth_event(langToSynth,textToSynth[len(sporadic):].encode('utf-8')).getSound(),(textToSynth.encode('utf-8')+"_"+langToSynth+dotwav).lower()]
         if gradint.got_program("lame"):
             # we can MP3-encode it (TODO make this optional)
             n = tm[0][:-len(dotwav)]+dotmp3
@@ -167,10 +178,10 @@ if toMove: sys.stderr.write("Renaming\n")
 for tmpfile,dest in toMove:
     oldDest = dest
     try:
-        os.rename(tmpfile,gradint.synthCache+os.sep+dest)
+        rename(tmpfile,gradint.synthCache+os.sep+dest)
     except OSError: # not a valid filename
         while gradint.fileExists(gradint.synthCache+os.sep+("__file%d" % count)+dotwav) or gradint.fileExists(gradint.synthCache+os.sep+("__file%d" % count)+dotmp3): count += 1
-        os.rename(tmpfile,gradint.synthCache+os.sep+("__file%d" % count)+dotwav)
+        rename(tmpfile,gradint.synthCache+os.sep+("__file%d" % count)+dotwav)
         open(gradint.synthCache+os.sep+gradint.transTbl,"ab").write("__file%d%s %s\n" % (count,dotwav,dest))
         dest = "__file%d%s" % (count,dotwav)
     if testMode:

samples/utils/trace.py

+#!/usr/bin/env python2
+
+# trace.py: script to generate raytraced animations of Gradint lessons
+# Version 1.32 (c) 2018-19,2021 Silas S. Brown.  License: GPL
+
+#  The Disney Pixar film "Inside Out" (2015) represented
+#  memories as spheres.  I don't have their CGI models, but
+#  we can do spheres in POV-Ray and I believe that idea is
+#  simple enough to be in the public domain (especially if
+#  NOT done like Pixar did it) - hopefully this might show
+#  some people how Gradint's method is supposed to work
+#  (especially if they've seen the Inside Out film).
+
+# This script generates the POV-Ray scenes from a lesson.
+# Gradint is run normally (passing any extra command-line arguments on,
+# must include outputFile so audio can be included in the animation)
+# and then the animation is written to /tmp/gradint.mp4.
+
+# Optionally add a static image representing each word (image will be
+# placed onto the spheres, and projected onto the back wall
+# when that word is being recalled)
+# e.g. word1_en.wav, word1_zh.wav, word1.jpg
+# (or png or gif).
+
+# Optionally add an mp4 video of a word in a particular language
+# e.g. word1_en.mp4 (probably best synchronised to word1_en.wav),
+# can also do this for commentsToAdd and orderlessCommentsToAdd files
+
+# Requires POV-Ray, ffmpeg, and the Python packages vapory
+# and futures (use sudo pip install futures vapory) -
+# futures is used to run multiple instances of POV-Ray on
+# multi-core machines.
+
+from optparse import OptionParser
+parser = OptionParser()
+parser.add_option("--fps",default=15,dest="theFPS",
+                  help="Frames per second (10 is insufficient for fast movement, so recommend at least 15)")
+parser.add_option("--res",default=480,
+                  help="Y-resolution: 240=NTSC VCD, 288=PAL VCD, 480=DVD, 607=WeChat channel, 720=Standard HD (Blu-Ray), 1080=Full HD (Blu-Ray)")
+parser.add_option("--translucent",action="store_true",default=False,dest="translucent_spheres_when_picture_visible",
+                  help="Translucent spheres when picture visible (slows down rendering but is better quality)")
+parser.add_option("--minutes",default=0,
+                  help="Maximum number of minutes to render (0 = unlimited, the default; can limit for test runs)")
+parser.add_option("--quality",default=9,dest="povray_quality",
+                  help="POVRay quality setting, default 9: 1=ambient light only, 2=lighting, 4,5=shadows, 8=reflections 9-11=radiosity etc")
+
+options, args = parser.parse_args()
+globals().update(options.__dict__)
+theFPS,res,minutes,povray_quality = int(theFPS),int(res),int(minutes),int(povray_quality)
+
+if res in [240,288]:
+    width_height_antialias = (352,res,0.3) # VCD.  antialias=None doesn't look very good at 300x200, cld try it at higher resolutions (goes to the +A param, PovRay default is 0.3 if -A specified without param; supersample (default 9 rays) if colour differs from neighbours by this amount)
+elif res==480: width_height_antialias = (640,480,0.001) # 480p (DVD)
+elif res==607: width_height_antialias = (1080,607,None) # WeChat Channels
+elif res==720: width_height_antialias = (1280,720,None) # Standard HD (Blu-Ray)
+elif res==1920: width_height_antialias = (1920,1080,None) # Full HD (Blu-Ray)
+else: raise Exception("Unknown vertical resolution specified: "+repr(res))
+debug_frame_limit = minutes * theFPS * 60
+
+import sys,os,traceback
+oldName = __name__ ; from vapory import * ; __name__ = oldName
+from concurrent.futures import ProcessPoolExecutor
+
+assert os.path.exists("gradint.py"), "You must move trace.py to the top-level Gradint directory and run it from there"
+sys.argv = [sys.argv[0]]+args
+import gradint
+assert gradint.outputFile, "You must run trace.py with gradint parameters that include outputFile"
+
+try: xrange
+except: xrange = range
+S,B = gradint.S,gradint.B
+
+class MovableParam:
+    def __init__(self): self.fixed = []
+    def fixAt(self,t,value):
+        while any(x[0]==t and not x[1]==value for x in self.fixed): t += 0.2
+        self.fixed.append((t,value))
+    def getPos(self,t):
+        assert self.fixed, "Should fixAt before getPos"
+        self.fixed.sort()
+        for i in xrange(len(self.fixed)):
+            if self.fixed[i][0] >= t:
+                if i: # interpolate
+                    if self.fixed[i-1][1]==None: return None
+                    duration = self.fixed[i][0]-self.fixed[i-1][0]
+                    progress = t-self.fixed[i-1][0]
+                    return (self.fixed[i][1]*progress + self.fixed[i-1][1]*(duration-progress))*1.0/duration
+                else: return self.fixed[i][1] # start position
+        return self.fixed[-1][1]
+
+class MovablePos:
+    def __init__(self): self.x,self.y,self.z = MovableParam(),MovableParam(),MovableParam()
+    def fixAt(self,t,*args):
+        if args[0]==None: x=y=z=None
+        else: x,y,z = args
+        self.x.fixAt(t,x),self.y.fixAt(t,y),self.z.fixAt(t,z)
+    def getPos(self,t):
+        r=(self.x.getPos(t),self.y.getPos(t),self.z.getPos(t))
+        if r==(None,None,None): return None
+        else: return r
+
+SceneObjects = set()
+
+class MovableSphere(MovablePos):
+    def __init__(self,radius=0.5,colour="prompt",imageFilename=None):
+        MovablePos.__init__(self)
+        self.colour = colour
+        self.imageFilename = imageFilename
+        self.radius = MovableParam()
+        self.radius.fixAt(-1,radius)
+        SceneObjects.add(self)
+    # fixAt(t,x,y,z) inherited
+    def obj(self,t):
+        pos = self.getPos(t)
+        if not pos: return # not in scene at this time
+        r = self.radius.getPos(t)
+        if self.imageFilename:
+            if translucent_spheres_when_picture_visible and bkgScrFade.getPos(t) < 1: transmittence = 0.5
+            else: transmittence = 0.3
+            img = wallPic(t,self.imageFilename) # if a video is playing whose key image matches ours, 'back-copy' the video frame (TODO: do this only on the correct L1 or L2 sphere?)
+            if not img: img = self.imageFilename
+            return Sphere(list(pos),r,colour(self.colour,t),Texture(Pigment(ImageMap('"'+S(img)+'"',"once","interpolate 2","transmit all "+str(transmittence)),'scale',[1.5*r,1.5*r,1],'translate',list(pos),'translate',[-.75*r,-.75*r,0])))
+        else: return Sphere(list(pos),r,colour(self.colour,t))
+
+class ObjCollection:
+    def __init__(self): self.objs = set()
+    def add(self,obj,dx,dy,dz): self.objs.add((obj,dx,dy,dz))
+    def get(self,dx,dy,dz): # should be small so:
+        for o,ddx,ddy,ddz in self.objs:
+            if (ddx,ddy,ddz) == (dx,dy,dz): return o
+    def fixAt(self,t,*args):
+        if args[0]==None: x=y=z=None
+        else: x,y,z = args
+        for obj,dx,dy,dz in self.objs:
+            if args==[None]: obj.fixAt(t,None,None,None)
+            else: obj.fixAt(t,x+dx,y+dy,z+dz)
+
+eventTrackers = {}
+def EventTracker(rowNo,imageFilename=None):
+    if not rowNo in eventTrackers:
+        eventTrackers[rowNo] = ObjCollection()
+        eventTrackers[rowNo].add(MovableSphere(1,"l1",imageFilename),-1,0,0)
+        eventTrackers[rowNo].add(MovableSphere(1,"l2",imageFilename),+1,0,0)
+        eventTrackers[rowNo].numRepeats = 0
+    return eventTrackers[rowNo]
+rCache = {}
+def repeatSphere(rowNo,numRepeats=0):
+    if not (rowNo,numRepeats) in rCache:
+        rCache[(rowNo,numRepeats)] = MovableSphere(0.1,"prompt")
+    return rCache[(rowNo,numRepeats)]
+def addRepeat(rowNo,t=0,length=0):
+    et = EventTracker(rowNo)
+    rpt = repeatSphere(rowNo,et.numRepeats)
+    if length:
+        rpt.fixAt(-1,None) # not exist yet (to save a tiny bit of POVRay computation)
+        rpt.fixAt(t-1,4*rowNo+1,0,61) # behind far wall
+        rpt.fixAt(t,4*rowNo-1,0,0) # ready to be 'batted'
+        et.fixAt(t,4*rowNo,0,10) # we're at bottom
+        camera_lookAt.fixAt(t,4*rowNo,0,10)
+        camera_lookAt.fixAt(t+length,4*rowNo,10,10)
+        camera_position.x.fixAt(t+length/2.0,4*rowNo)
+        # careful with Y : try to avoid sudden vertical motion between 2 sequences
+        camera_position.y.fixAt(t+length*.2,1)
+        camera_position.y.fixAt(t+length*.8,4)
+        camera_position.z.fixAt(t+length*.2,-10)
+        camera_position.z.fixAt(t+length*.8,-5)
+    et.add(rpt,0,1+0.2*et.numRepeats,0) # from now on we keep this marker
+    et.fixAt(t+length,4*rowNo,10,10) # at end of repeat (or at t=0) we're at top, and the repeat marker is in place
+    et.numRepeats += 1
+
+camera_position = MovablePos()
+camera_lookAt = MovablePos()
+def cam(t): return Camera('location',list(camera_position.getPos(t)),'look_at',list(camera_lookAt.getPos(t)))
+def lights(t): return [LightSource([camera_position.x.getPos(t)+10, 15, -20], [1.3, 1.3, 1.3])]
+
+background_screen = [] # (startTime,endTime,pictureName,pictureActual)
+background_screen_size = 50
+bkgScrFade = MovableParam() ; bkgScrFade.fixAt(-1,1)
+bkgScrX = MovableParam()
+def wallPic(t,ifImg=None):
+    if bkgScrFade.getPos(t) == 1: return # no picture if we're faded out
+    found = None
+    for st,et,img,pic in background_screen:
+        if st <= t: found = (st,et,img,pic)
+        elif st > t: break
+    if found:
+        st,et,img,pic = found
+        if ifImg and not img==ifImg: return
+        if B(pic).endswith(B(os.extsep+"mp4")):
+            # need to take single frame
+            T = min(t,et-1.0/theFPS)-st # don't go past last frame
+            out = B(pic)[:-4]+B("-"+str(T)+os.extsep+"jpg")
+            while T > 0 and not os.path.exists(out): # (TODO: if its frame rate is low enough, we might already have the same frame even at a slightly different T)
+                cmd = "ffmpeg -n -threads 1 -accurate_seek -ss "+str(T)+" -i "+S(pic)+" -vframes 1 -q:v 1 "+S(out)+" </dev/null >/dev/null"
+                print (cmd)
+                os.system(cmd)
+                T -= 1.0/theFPS
+            if os.path.exists(out): return out
+            else: return None
+        else: return pic
+def wall(t):
+    picToUse = wallPic(t)
+    if picToUse: return [Plane([0, 0, 1], 60, Texture(Pigment('color', [1, 1, 1])), Texture(Pigment(ImageMap('"'+S(picToUse)+'"',"once","transmit all "+str(bkgScrFade.getPos(t))),'scale',[background_screen_size,background_screen_size,1],'translate',[bkgScrX.getPos(t)-background_screen_size/2,0,0])), Finish('ambient',0.9))]
+    else: return [Plane([0, 0, 1], 60, Texture(Pigment('color', [1, 1, 1])), Finish('ambient',0.9))] # TODO: why does this look brighter than with ImageMap at transmit all 1.0 ?
+
+ground = Plane( [0, 1, 0], -1, Texture( Pigment( 'color', [1, 1, 1]), Finish( 'phong', 0.1, 'reflection',0.4, 'metallic', 0.3))) # from vapory example
+
+def colour(c,t=None):
+    c = {"l1":[.8,1,.2],"l2":[.5,.5,.9],"prompt":[1,.6,.5]}[c] # TODO: better colours
+    if translucent_spheres_when_picture_visible and not t==None and bkgScrFade.getPos(t) < 1: return Texture(Pigment('color',c,'filter',0.7))
+    else: return Texture(Pigment('color',c))
+def scene(t):
+    """ Returns the scene at time 't' (in seconds) """
+    return Scene(cam(t), lights(t) + wall(t) + [ground] + [o for o in [x.obj(t) for x in SceneObjects] if not o==None])
+
+def Event_draw(self,startTime,rowNo,inRepeat): pass
+gradint.Event.draw = Event_draw
+
+def CompositeEvent_draw(self,startTime,rowNo,inRepeat):
+  if self.eventList:
+    t = startTime
+    for i in self.eventList:
+      i.draw(t,rowNo,True)
+      t += i.length
+    if inRepeat: return
+    # Call addRepeat, but postpone the start until the
+    # first loggable event, to reduce rapid camera mvt
+    st0 = startTime
+    for i in self.eventList:
+      if i.makesSenseToLog(): break
+      else: startTime += i.length
+    if startTime==t: startTime = st0 # shouldn't happen
+    addRepeat(rowNo,startTime,t-startTime)
+gradint.CompositeEvent.draw=CompositeEvent_draw
+
+def Event_colour(self,language):
+    if self.makesSenseToLog():
+      if language==gradint.firstLanguage: return "l1"
+      else: return "l2"
+    else: return "prompt"
+gradint.Event.colour = Event_colour
+
+def eDraw(startTime,length,rowNo,colour):
+    minR = 0.5
+    if colour in ["l1","l2"]:
+        if colour=="l1": delta = -1
+        else: delta = +1
+        et = EventTracker(rowNo).get(delta,0,0)
+        r = et.radius
+        if hasattr(et,"imageFilename"):
+            background_screen.append((startTime,startTime+length,et.imageFilename,et.imageFilename))
+            bkgScrX.fixAt(startTime,4*rowNo)
+            bkgScrX.fixAt(startTime+length,4*rowNo)
+    else:
+        r = repeatSphere(rowNo,EventTracker(rowNo).numRepeats).radius
+        minR = 0.1
+    maxR = min(max(length,minR*1.5),minR*3) # TODO: vary with event's volume, so cn see the syllables? (partials can do that anyway)
+    r.fixAt(startTime,minR)
+    r.fixAt(startTime+length,minR)
+    if length/2.0 > 0.5:
+        r.fixAt(startTime+0.5,maxR)
+        # TODO: wobble in the middle?
+        r.fixAt(startTime+length-0.5,maxR)
+    else: r.fixAt(startTime+length/2.0,maxR)
+
+def SampleEvent_draw(self,startTime,rowNo,inRepeat):
+    if B(self.file).startswith(B(gradint.partialsDirectory)): l=B(self.file).split(B(os.sep))[1]
+    else: l = gradint.languageof(self.file)
+    eDraw(startTime,self.length,rowNo,self.colour(S(l)))
+gradint.SampleEvent.draw = SampleEvent_draw
+def SynthEvent_draw(self,startTime,rowNo,inRepeat): eDraw(startTime,self.length,rowNo,self.colour(self.language))
+gradint.SynthEvent.draw = SynthEvent_draw
+
+def chkImg(i):
+    if not "_" in S(i.file): return
+    for imgExt in ["gif","png","jpeg","jpg"]:
+        imageFilename = B(i.file)[:B(i.file).rindex(B("_"))]+B(os.extsep+imgExt) # TODO: we're assuming no _en etc in the image filename (projected onto both L1 and L2)
+        if os.path.exists(imageFilename):
+            return os.path.abspath(imageFilename)
+
+def runGradint():
+  gradint.gluedListTracker=[]
+  gradint.waitBeforeStart=0
+  gradint.main()
+  gradint.gluedListTracker.sort(key=lambda e:e[0].glue.length+e[0].glue.adjustment)
+  duration = 0
+  for l,row in zip(gradint.gluedListTracker,xrange(len(gradint.gluedListTracker))):
+    def check_for_pictures():
+     for gluedEvent in l:
+      event = gluedEvent.event
+      try: el=event.eventList
+      except: el=[event]
+      for j in el:
+       try: el2=j.eventList
+       except: el2=[j]
+       for i in el2:
+        if hasattr(i,"file") and B("_") in B(i.file):
+            imageFilename = chkImg(i)
+            if imageFilename:
+                return EventTracker(row,imageFilename)
+    check_for_pictures()
+    if hasattr(l[0],"timesDone"): timesDone = l[0].timesDone
+    else: timesDone = 0
+    for i in xrange(timesDone): addRepeat(row)
+    glueStart = 0
+    for i in l:
+      i.event.draw(i.getEventStart(glueStart),row,False)
+      glueStart = i.getAdjustedEnd(glueStart)
+      duration = max(duration,glueStart)
+  for t,e in gradint.lastLessonMade.events: # check for videos
+      if hasattr(e,"file") and hasattr(e,"exactLen"):
+          video = B(e.file)[:B(e.file).rindex(B(os.extsep))]+B(os.extsep+"mp4")
+          if os.path.exists(video): # overwrite static image while playing
+              i,v = chkImg(e),os.path.abspath(video)
+              if not i: i=v
+              background_screen.append((t,t+e.exactLen,i,v))
+  background_screen.sort()
+  i = 0 # more items might be inserted, so don't use range here
+  while i < len(background_screen)-1:
+      if background_screen[i][1] > background_screen[i+1][1]: # overlap: we end after next one ends: insert a jump-back-to-us after
+          background_screen.insert(i+2,(background_screen[i+1][1],background_screen[i][1],background_screen[i][2],background_screen[i][3])) # restore old after new one ends
+      if background_screen[i][1] > background_screen[i+1][0] and background_screen[i][0] < background_screen[i+1][0]: # overlap: we end after next one starts, but we start before it starts
+          background_screen[i] = (background_screen[i][0],background_screen[i+1][0],background_screen[i][2],background_screen[i][3]) # new one takes precedence
+      if background_screen[i][0]==background_screen[i+1][0]: # equal start, but next one might be longer
+          background_screen[i+1]=(background_screen[i][1],background_screen[i+1][1],background_screen[i+1][2],background_screen[i+1][3])
+      if background_screen[i][2]==background_screen[i+1][2] and background_screen[i][1]+5>=background_screen[i+1][0] and background_screen[i][1] < background_screen[i+1][0]:
+          # avoid turning off for 5 seconds or less if showing the same image (or a video of it)
+          background_screen.insert(i+1,(background_screen[i][1],background_screen[i+1][0],background_screen[i][2],background_screen[i][2])) # just the image
+      i += 1
+  for i in xrange(len(background_screen)):
+      startTime,endTime,picName,img = background_screen[i]
+      if i and startTime > background_screen[i-1][1] + 0.5:
+          bkgScrFade.fixAt(startTime,1) # start faded out
+      # else (less than 0.5sec between images) don't try to start faded out
+      fadeOutTime = endTime
+      if i<len(background_screen)-1:
+          if endTime + 0.5 > background_screen[i+1][0]:
+              fadeOutTime = None # as above (< 0.5sec between images)
+          else: fadeOutTime = max(fadeOutTime,min(background_screen[i+1][0]-1,fadeOutTime+5))
+          if not fadeOutTime == None:
+              # don't move the screen during any extended fade-out:
+              for ii in xrange(len(bkgScrX.fixed)):
+                  if bkgScrX.fixed[ii][0]==endTime:
+                      bkgScrX.fixed[ii]=((fadeOutTime,bkgScrX.fixed[ii][1]))
+                      break
+      if not fadeOutTime==None: bkgScrFade.fixAt(fadeOutTime,1)
+      if endTime >= startTime+0.5:
+          bkgScrFade.fixAt(startTime+0.5,0.3)
+          bkgScrFade.fixAt(endTime-0.5,0.3)
+      else:
+          bkgScrFade.fixAt((startTime+endTime)/2.0,0.3)
+  return duration
+
+def tryFrame(f):
+    frame,numFrames = f
+    print ("Making frame "+str(frame)+" of "+str(numFrames))
+    try:
+        try: os.mkdir("/tmp/"+repr(frame)) # vapory writes a temp .pov file and does not change its name per process, so better be in a process-unique directory
+        except: pass
+        os.chdir("/tmp/"+repr(frame))
+        scene(frame*1.0/theFPS).render(width=width_height_antialias[0], height=width_height_antialias[1], antialiasing=width_height_antialias[2], quality=povray_quality, outfile="/tmp/frame%05d.png" % frame)
+        # TODO: TURN OFF JITTER with -J if using anti-aliasing in animations
+        os.chdir("/tmp") ; os.system('rm -r '+repr(frame))
+        return None
+    except:
+        if frame==0: raise
+        traceback.print_exc()
+        sys.stderr.write("Frame %d render error, will skip\n" % frame)
+        return "cp /tmp/frame%05d.png /tmp/frame%05d.png" % (frame-1,frame)
+
+def main():
+    executor = ProcessPoolExecutor()
+    duration = runGradint()
+    numFrames = int(duration*theFPS)
+    if debug_frame_limit: numFrames=min(numFrames,debug_frame_limit)
+    # TODO: pickle all MovableParams so can do the rendering on a different machine than the one that makes the Gradint lesson?
+    for c in list(executor.map(tryFrame,[(frame,numFrames) for frame in xrange(numFrames)]))+[
+        "ffmpeg -nostdin -y -framerate "+repr(theFPS)+" -i /tmp/frame%05d.png -i "+gradint.outputFile+" -movflags faststart -pix_fmt yuv420p -filter_complex tpad=stop=-1:stop_mode=clone -shortest /tmp/gradint.mp4 && if [ -d /Volumes ]; then open /tmp/gradint.mp4; fi" #  (could alternatively run with -vcodec huffyuv /tmp/gradint.avi for lossless, insead of --movflags etc, but will get over 6 gig and may get A/V desync problems in mplayer/VLC that -delay doesn't fix, however -b:v 1000k seems to look OK; for WeChat etc you need to recode to h.264, and for HTML 5 video need recode to WebM (but ffmpeg -c:v libvpx no good if not compiled with support for those libraries; may hv to convert on another machine i.e. ffmpeg -i gradint.mp4 -vf scale=320:240 -c:v libvpx -b:v 500k gradint.webm))
+        ]:
+        if c: # patch up skipped frames, then run ffmpeg
+            print (c) ; os.system(c)
+    for f in xrange(numFrames): os.remove("/tmp/frame%05d.png" % f) # wildcard from command line could get 'argument list too long' on BSD etc
+if __name__=="__main__": main()
+else: print (__name__)

gradint-build/samples/utils/transliterate.py → samples/utils/transliterate.py

+#!/usr/bin/env python2
+
 # transliterate.py - print a 2nd-language-transliterated version of vocab.txt and any .txt pairs in samples
 # (may be useful for grepping, loading to Latin-only PDA, etc)
 # (note: leaves comments untransliterated, + may not translit all text if gradint is set up so a transliterating synth will not be used)

gradint-build/server/README.txt → server/README.txt

@@ -18,3 +18,5 @@ samples.cgi - CGI script to browse a samples directory
 or that the site is not publically viewable)
 
 espeak.cgi - script that lets a Web user play with espeak options
+
+Other files - see description at the top of the file

server/cantonese.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#  (should work with either Python 2 or Python 3)
+
+# cantonese.py - Python functions for processing Cantonese transliterations
+# (uses eSpeak and Gradint for help with some of them)
+
+# v1.48 (c) 2013-15,2017-24 Silas S. Brown.  License: GPL
+
+cache = {} # to avoid repeated eSpeak runs,
+# zi -> jyutping or (pinyin,) -> translit
+dryrun_mode = False # True = prepare to populate cache in batch
+jyutping_dryrun,pinyin_dryrun = set(),set()
+
+import re, pickle, os, sys
+if '--cache' in sys.argv:
+  cache_fname = sys.argv[sys.argv.index('--cache')+1]
+else: cache_fname = os.environ.get("JYUTPING_CACHE","/tmp/.jyutping-cache")
+try: cache = pickle.Unpickler(open(cache_fname,"rb")).load()
+except: pass
+
+extra_zhy_dict = { # TODO: add these to the real zhy_list in eSpeak
+  u"\u9c85":"bat3",u"\u9b81":"bat3",
+}
+
+def S(v): # make sure it's a string in both Python 2 and 3
+  if type("")==type(u""): # Python 3
+    try: return v.decode('utf-8') # in case it's bytes
+    except: return v
+  else: return v
+def B(v): # make sure it's bytes in Python 3, str in Python 2
+  if type(v)==type(u""): return v.encode('utf-8')
+  return v
+
+def get_jyutping(hanzi,mustWork=1):
+  if not type(hanzi)==type(u""): hanzi=hanzi.decode('utf-8')
+  for k,v in extra_zhy_dict.items(): hanzi=hanzi.replace(k,v)
+  global espeak
+  if not espeak:
+      espeak = import_gradint().ESpeakSynth()
+      if not espeak.works_on_this_platform(): # must call
+          raise Exception("espeak.works_on_this_platform")
+      assert espeak.supports_language("zhy")
+
+  global jyutping_dryrun
+  if dryrun_mode:
+      if not hanzi in cache: jyutping_dryrun.add(hanzi)
+      return "aai1" # placeholder value
+  elif jyutping_dryrun:
+      jyutping_dryrun = list(jyutping_dryrun)
+      vals = espeak.transliterate_multiple("zhy",jyutping_dryrun,0)
+      assert len(jyutping_dryrun)==len(vals)
+      for k,v in zip(jyutping_dryrun,vals):
+        cache[k]=S(v).replace("7","1").lower() # see below
+      jyutping_dryrun = set()
+  if hanzi in cache: jyutping = cache[hanzi]
+  else: cache[hanzi] = jyutping = S(espeak.transliterate("zhy",hanzi,forPartials=0)).replace("7","1").lower() # .lower() needed because espeak sometimes randomly capitalises e.g. 2nd hanzi of 'hypocrite' (Mandarin xuwei de ren)
+  if mustWork: assert jyutping.strip(), "No translit. result for "+repr(hanzi)
+  elif not jyutping.strip(): jyutping=""
+  return jyutping
+espeak = 0
+
+def hanzi_only(unitext): return u"".join(filter(lambda x:0x4e00<=ord(x)<0xa700 or ord(x)>=0x10000, list(unitext)))
+def py2nums(pinyin):
+  if not type(pinyin)==type(u""):
+    pinyin = pinyin.decode('utf-8')
+  if not pinyin.strip(): return ""
+  global pinyin_dryrun
+  if pinyin_dryrun:
+    pinyin_dryrun = list(pinyin_dryrun)
+    vals = espeak.transliterate_multiple("zh",pinyin_dryrun,0)
+    assert len(pinyin_dryrun)==len(vals)
+    for i in range(len(pinyin_dryrun)):
+      cache[(pinyin_dryrun[i],)]=vals[i]
+    pinyin_dryrun = set()
+  if (pinyin,) in cache: pyNums = cache[(pinyin,)]
+  else: pyNums = espeak.transliterate("zh",pinyin,forPartials=0) # (this transliterate just does tone marks to numbers, adds 5, etc; forPartials=0 because we DON'T want to change letters like X into syllables, as that won't happen in jyutping and we're going through it tone-by-tone)
+  assert pyNums and pyNums.strip(), "espeak.transliterate returned %s for %s" % (repr(pyNums),repr(pinyin))
+  return re.sub("a$","a5",re.sub("(?<=[a-zA-Z])er([1-5])",r"e\1r5",S(pyNums)))
+if type(u"")==type(""): # Python 3
+  getNext = lambda gen: gen.__next__()
+else: getNext = lambda gen: gen.next()
+def adjust_jyutping_for_pinyin(hanzi,jyutping,pinyin):
+  # If we have good quality (proof-read etc) Mandarin pinyin, this can sometimes improve the automatic Cantonese transcription
+  if not type(hanzi)==type(u""): hanzi = hanzi.decode('utf-8')
+  hanzi = hanzi_only(hanzi)
+  if not re.search(py2j_chars,hanzi): return jyutping
+  pinyin = re.findall('[A-Za-z]*[1-5]',py2nums(pinyin))
+  if not len(pinyin)==len(hanzi): return jyutping # can't fix
+  jyutping = S(jyutping)
+  i = 0 ; tones = re.finditer('[1-7]',jyutping) ; j2 = []
+  for h,p in zip(list(hanzi),pinyin):
+    try: j = getNext(tones).end()
+    except StopIteration: return jyutping # one of the hanzi has no Cantonese reading in our data: we'll warn "failed to fix" below
+    j2.append(jyutping[i:j]) ; i = j
+    if h in py2j and p.lower() in py2j[h]: j2[-1]=j2[-1][:re.search("[A-Za-z]*[1-7]$",j2[-1]).start()]+py2j[h][p.lower()]
+  return "".join(j2)+jyutping[i:]
+py2j={
+u"\u4E2D":{"zhong1":"zung1","zhong4":"zung3"},
+u"\u4E3A\u70BA":{"wei2":"wai4","wei4":"wai6"},
+u"\u4E50\u6A02":{"le4":"lok6","yue4":"ngok6"},
+u"\u4EB2\u89AA":{"qin1":"can1","qing4":"can3"},
+u"\u4EC0":{"shen2":"sam6","shi2":"sap6"}, # unless zaap6
+u"\u4F20\u50B3":{"chuan2":"cyun4","zhuan4":"zyun6"},
+u"\u4FBF":{"bian4":"bin6","pian2":"pin4"},
+u"\u5047":{"jia3":"gaa2","jia4":"gaa3"},
+u"\u5174\u8208":{"xing1":"hing1","xing4":"hing3"},
+# u"\u5207":{"qie4":"cai3","qie1":"cit3"}, # WRONG (rm'd v1.17).  It's cit3 in re4qie4.  It just wasn't in yiqie4 (which zhy_list has as an exception anyway)
+u"\u521B\u5275":{"chuang1":"cong1","chuang4":"cong3"},
+u"\u53EA":{"zhi1":"zek3","zhi3":"zi2"},
+u"\u53F7\u865F":{"hao4":"hou6","hao2":"hou4"},
+u"\u548C":{"he2":"wo4","he4":"wo6"},
+u"\u54BD":{"yan1":"jin1","yan4":"jin3","ye4":"jit3"},
+u"\u5708":{"juan4":"gyun6","quan1":"hyun1"},
+u"\u597D":{"hao3":"hou2","hao4":"hou3"},
+u"\u5C06\u5C07":{"jiang1":"zoeng1","jiang4":"zoeng3"},
+u"\u5C11":{"shao3":"siu2","shao4":"siu3"},
+u"\u5DEE":{"cha4":"caa1","cha1":"caa1","chai1":"caai1"},
+u"\u5F37\u5F3A":{"qiang2":"koeng4","qiang3":"koeng5"},
+u"\u62C5\u64D4":{"dan1":"daam1","dan4":"daam3"},
+u"\u6323\u6399":{"zheng4":"zaang6","zheng1":"zang1"},
+u"\u6570\u6578":{"shu3":"sou2","shu4":"sou3"},
+u"\u671D":{"chao2":"ciu4","zhao1":"ziu1"},
+u"\u6ED1":{"hua2":"waat6","gu3":"gwat1"},
+u"\u6F02":{"piao1":"piu1","piao3 piao4":"piu3"},
+u"\u76DB":{"sheng4":"sing6","cheng2":"sing4"},
+u"\u76F8":{"xiang1":"soeng1","xiang4":"soeng3"},
+u"\u770B":{"kan4":"hon3","kan1":"hon1"},
+u"\u79CD\u7A2E":{"zhong3":"zung2","zhong4":"zung3"},
+u"\u7EF7\u7E43":{"beng1":"bang1","beng3":"maang1"},
+u"\u8208":{"xing1":"hing1","xing4":"hing3"},
+u"\u843D":{"luo1 luo4 lao4":"lok6","la4":"laai6"},
+u"\u8457":{"zhu4":"zyu3","zhuo2":"zoek3","zhuo2 zhao2 zhao1 zhe5":"zoek6"},
+u"\u8981":{"yao4":"jiu3","yao1":"jiu1"},
+u"\u89C1\u898B":{"jian4":"gin3","xian4":"jin6"},
+u"\u89C9\u89BA":{"jue2":"gok3","jiao4":"gaau3"},
+u"\u8B58\u8BC6":{"shi2 shi4":"sik1","zhi4":"zi3"},
+u"\u8ABF\u8C03":{"diao4":"diu6","tiao2":"tiu4"},
+u"\u91CF":{"liang2":"loeng4","liang4":"loeng6"},
+u"\u9577\u957F":{"chang2":"coeng4","zhang3":"zoeng2"},
+u"\u9593\u95F4":{"jian1":"gaan1","jian4":"gaan3"},
+u"\u96BE\u96E3":{"nan2":"naan4","nan4":"naan6"}}
+for k in list(py2j.keys()):
+   if len(k)>1:
+    for c in list(k): py2j[c]=py2j[k]
+    del py2j[k]
+for _,v in py2j.items():
+  for k in list(v.keys()):
+    if len(k.split())>1:
+      for w in k.split(): v[w]=v[k]
+      del v[k]
+py2j_chars = re.compile(u'['+''.join(list(py2j.keys()))+']')
+
+def jyutping_to_lau(j):
+  j = S(j).lower().replace("j","y").replace("z","j")
+  for k,v in jlRep: j=j.replace(k,v)
+  return j.lower().replace("ohek","euk")
+def jyutping_to_lau_java(jyutpingNo=2,lauNo=1):
+  # for annogen.py 3.29+ --annotation-postprocess to ship Jyutping and generate Lau at runtime
+  return 'if(annotNo=='+str(jyutpingNo)+'||annotNo=='+str(lauNo)+'){m=Pattern.compile("<rt>(.*?)</rt>").matcher(r);sb=new StringBuffer();while(m.find()){String r2=(annotNo=='+str(jyutpingNo)+'?m.group(1).replaceAll("([1-7])(.)","$1&shy;$2"):(m.group(1)+" ").toLowerCase().replace("j","y").replace("z","j")'+''.join('.replace("'+k+'","'+v+'")' for k,v in jlRep)+'.toLowerCase().replace("ohek","euk").replaceAll("([1-7])","<sup>$1</sup>-").replace("- "," ").replaceAll(" $","")),tmp=m.group(1).substring(0,1);if(annotNo=='+str(lauNo)+'&&tmp.equals(tmp.toUpperCase()))r2=r2.substring(0,1).toUpperCase()+r2.substring(1);m.appendReplacement(sb,"<rt>"+r2+"</rt>");}m.appendTail(sb); r=sb.toString();}' # TODO: can probably go faster with mapping for some of this
+def incomplete_lau_to_jyutping(l):
+  # incomplete: assumes Lau didn't do the "aa" -> "a" rule
+  l = S(l).lower().replace("euk","ohek")
+  for k,v in ljRep: l=l.replace(k,v)
+  return l.lower().replace("j","z").replace("y","j")
+def incomplete_lau_to_yale_u8(l): return jyutping_to_yale_u8(incomplete_lau_to_jyutping(l))
+jlRep = [(unchanged,unchanged.upper()) for unchanged in "aai aau aam aang aan aap aat aak ai au am ang an ap at ak a ei eng ek e iu im ing in ip it ik i oi ong on ot ok ung uk".split()] + [("eoi","UI"),("eon","UN"),("eot","UT"),("eok","EUK"),("oeng","EUNG"),("oe","EUH"),("c","ch"),("ou","O"),("o","OH"),("yu","UE"),("u","OO")]
+jlRep.sort(key=lambda a:-len(a[0])) # longest 1st
+# u to oo includes ui to ooi, un to oon, ut to oot
+# yu to ue includes yun to uen and yut to uet
+# drawing from the table on http://www.omniglot.com/writing/cantonese.htm plus this private communication:
+# Jyutping "-oeng" maps to Sidney Lau "-eung".
+# Jyutping "jyu" maps to Sidney Lau "yue". (consequence of yu->ue, j->y)
+ljRep=[(b.lower(),a.upper()) for a,b in jlRep]
+ljRep.sort(key=lambda a:-len(a[0])) # longest 1st
+
+def ping_or_lau_to_syllable_list(j): return re.sub(r"([1-9])(?![0-9])",r"\1 ",re.sub(r"[!-/:-@^-`]"," ",S(j))).split()
+
+def hyphenate_ping_or_lau_syl_list(sList,groupLens=None):
+    if type(sList) in [str,type(u"")]:
+        sList = ping_or_lau_to_syllable_list(sList)
+    return hyphenate_syl_list(sList,groupLens)
+def hyphenate_yale_syl_list(sList,groupLens=None):
+    # (if sList is a string, the syllables must be space-separated,
+    #  which will be the case if to_yale functions below are used)
+    if not type(sList)==list: sList = sList.split()
+    return hyphenate_syl_list(sList,groupLens)
+def hyphenate_syl_list(sList,groupLens=None):
+    assert type(sList) == list
+    if '--hyphenate-all' in sys.argv: groupLens = [len(sList)]
+    elif not groupLens: groupLens = [1]*len(sList) # don't hyphenate at all if we don't know
+    else: assert sum(groupLens) == len(sList), "sum("+repr(groupLens)+")!=len("+repr(sList)+")"
+    r = [] ; start = 0
+    for g in groupLens:
+        r.append("-".join(S(x) for x in sList[start:start+g]))
+        start += g
+    return " ".join(r)
+    
+def jyutping_to_yale_TeX(j): # returns space-separated syllables
+  ret=[]
+  for syl in ping_or_lau_to_syllable_list(S(j).lower().replace("eo","eu").replace("oe","eu").replace("j","y").replace("yyu","yu").replace("z","j").replace("c","ch")):
+    vowel=lastVowel=None
+    for i in range(len(syl)):
+      if syl[i] in "aeiou":
+        vowel=i ; break
+    if vowel==None and re.match(r"h?(m|ng)[456]",syl): # standalone nasal syllables
+      vowel = syl.find('m')
+      if vowel<0: vowel = syl.index('n')
+      lastVowel = syl.find('g')
+      if lastVowel<0: lastVowel = vowel
+    if vowel==None:
+      ret.append(syl.upper()) ; continue # English word or letter in the Chinese?
+    if syl[vowel:vowel+2] == "aa" and (len(syl)<vowel+2 or syl[vowel+2] in "123456"):
+      syl=syl[:vowel]+syl[vowel+1:] # final aa -> a
+    # the tonal 'h' goes after all the vowels but before any consonants:
+    for i in range(len(syl)-1,-1,-1):
+      if syl[i] in "aeiou":
+        lastVowel=i ; break
+    if syl[-1] in "1234567":
+      # get_jyutping replaces 7 with 1 because zhy_list is
+      # more Canton-type than Hong Kong-type Cantonese and
+      # there is considerable disagreement on which "1"s
+      # should be "7"s, but if you pass any "7" into the
+      # jyutping_to_yale functions we can at least process
+      # it here:
+      tone = ["\=",r"\'","",r"\`",r"\'","",r"\`"][int(syl[-1])-1]
+      if syl[-1] in "456":
+        syl=syl[:lastVowel+1]+"h"+syl[lastVowel+1:]
+      ret.append((syl[:vowel]+tone+syl[vowel:-1]).replace(r"\=i",r"\=\i{}").replace(r"\=I",r"\=\I{}"))
+    else: ret.append(syl.upper()) # English word or letter in the Chinese?
+  return ' '.join(ret)
+
+def jyutping_to_yale_u8(j): # returns space-separated syllables
+  import unicodedata
+  def mysub(z,l):
+    for x,y in l:
+      z = re.sub(re.escape(x)+r"(.)",r"\1"+y,z)
+    return z
+  if type(u"")==type(""): U=str # Python 3
+  else: # Python 2
+    def U(x):
+      try: return x.decode('utf-8') # might be an emoji pass-through
+      except: return x # already Unicode
+  return unicodedata.normalize('NFC',mysub(U(jyutping_to_yale_TeX(j).replace(r"\i{}","i").replace(r"\I{}","I")),[(r"\`",u"\u0300"),(r"\'",u"\u0301"),(r"\=",u"\u0304")])).encode('utf-8')
+
+def superscript_digits_TeX(j):
+  # for jyutping and Sidney Lau
+  j = S(j)
+  for digit in "123456789": j=j.replace(digit,r"\raisebox{-0.3ex}{$^"+digit+r"$}\hspace{0pt}")
+  return j
+
+def superscript_digits_HTML(j):
+  j = S(j)
+  for digit in "123456789": j=j.replace(digit,"<sup>"+digit+"</sup>")
+  return j
+
+def superscript_digits_UTF8(j):
+  # WARNING: not all fonts have all digits; many have only the first 3.  superscript_digits_HTML might be better for browsers, even though it does produce more bytes.
+  j = S(j)
+  for digit in range(1,10): j=j.replace(str(digit),S(u"¹²³⁴⁵⁶⁷⁸⁹"[digit-1].encode('utf-8')))
+  if type(j)==type(u""): j=j.encode('utf-8') # Python 3
+  return j
+
+def import_gradint():
+    global gradint
+    try: return gradint
+    except: pass
+    # when importing gradint, make sure no command line
+    tmp,sys.argv = sys.argv,sys.argv[:1]
+    import gradint
+    sys.argv = tmp
+    gradint.espeak_preprocessors = {}
+    return gradint
+
+def do_song_subst(hanzi_u8): return B(hanzi_u8).replace(unichr(0x4f7f).encode('utf-8'),unichr(0x38c8).encode('utf-8')) # Mandarin shi3 (normally jyutping sai2) is usually si3 in songs, so substitute a rarer character that unambiguously has that reading before sending to get_jyutping
+
+if __name__ == "__main__":
+    # command-line use: output Lau for each line of stdin
+    # (or Yale if there's a --yale in sys.argv, or both
+    # with '#' separators if --yale#lau in sys.argv,
+    # also --yale#ping and --yale#lau#ping accepted);
+    # if there's a # in the line, assume it's hanzi#pinyin
+    # (for annogen.py --reannotator="##python cantonese.py")
+    lines = sys.stdin.read().replace("\r\n","\n").split("\n")
+    if lines and not lines[-1]: del lines[-1]
+    dryrun_mode = True
+    def songSubst(l):
+      if '--song-lyrics' in sys.argv: l=do_song_subst(l)
+      return l
+    for l in lines:
+      if '#' in l: l,pinyin = l.split('#')
+      else: pinyin = None
+      get_jyutping(songSubst(l))
+      if pinyin and not type(pinyin)==type(u""):
+        pinyin = pinyin.decode('utf-8')
+      if pinyin and not (pinyin,) in cache:
+        pinyin_dryrun.add(pinyin)
+        for w in pinyin.split():
+          for h in w.split('-'):
+            pinyin_dryrun.add(h)
+    dryrun_mode = False
+    for l in lines:
+      if '#' in l: l,pinyin = l.split('#')
+      else: pinyin = None
+      jyutping = get_jyutping(songSubst(l),0)
+      if not jyutping: groupLens = None # likely a Unihan-only 'fallback readings' zi that has no Cantonese
+      elif pinyin:
+        jyutping = adjust_jyutping_for_pinyin(l,jyutping,pinyin)
+        groupLens = [0]
+        for syl,space in re.findall('([A-Za-z]*[1-5])( *)',' '.join('-'.join(py2nums(h) for h in w.split('-')) for w in pinyin.split())): # doing it this way so we're not relying on espeak transliterate_multiple to preserve spacing and hyphenation
+          groupLens[-1] += 1
+          if space: groupLens.append(0)
+        if not groupLens[-1]: groupLens=groupLens[:-1]
+        lenWanted = len(ping_or_lau_to_syllable_list(jyutping))
+        if sum(groupLens) > lenWanted: # probably silent -r to drop
+          for i,word in enumerate(py2nums(pinyin).split()):
+            if re.search("[1-5]r5",word):
+              groupLens[i] -= 1
+              if sum(groupLens)==lenWanted: break
+        if not sum(groupLens)==lenWanted:
+          sys.stderr.write("WARNING: failed to fix "+pinyin+" ("+py2nums(pinyin)+") to "+jyutping+" ("+repr(ping_or_lau_to_syllable_list(jyutping))+") from "+l+", omitting\n")
+          groupLens = None ; jyutping = ""
+      else: groupLens = None
+      if "--yale#lau" in sys.argv: print (hyphenate_yale_syl_list(jyutping_to_yale_u8(jyutping),groupLens)+"#"+superscript_digits_HTML(hyphenate_ping_or_lau_syl_list(jyutping_to_lau(jyutping),groupLens)))
+      elif '--yale#ping' in sys.argv: print (hyphenate_yale_syl_list(jyutping_to_yale_u8(jyutping),groupLens)+"#"+jyutping.replace(' ',''))
+      elif "--yale#lau#ping" in sys.argv: print (hyphenate_yale_syl_list(jyutping_to_yale_u8(jyutping),groupLens)+"#"+superscript_digits_HTML(hyphenate_ping_or_lau_syl_list(jyutping_to_lau(jyutping),groupLens))+"#"+jyutping.replace(' ',''))
+      elif "--yale" in sys.argv: print (hyphenate_yale_syl_list(jyutping_to_yale_u8(jyutping),groupLens))
+      else: print (superscript_digits_HTML(hyphenate_ping_or_lau_syl_list(jyutping_to_lau(jyutping),groupLens)))
+    try: pickle.Pickler(open(cache_fname,"wb"),-1).dump(cache)
+    except: pass

gradint-build/server/email-lesson-archive.sh → server/email-lesson-archive.sh

 #!/bin/bash
 
 # email-lesson-archive.sh - archive an old email-lesson user
-# (C) 2008 Silas S. Brown, License: GPL
+# (C) 2008,2021-22 Silas S. Brown, License: GPL
 
 if ! pwd|grep email_lesson_users >/dev/null; then
   echo "This script should be run from an email_lesson_users directory (see email-lesson.sh)"
@@ -13,29 +13,29 @@ if test "a$1" == a; then
 fi
 . config
 while ! test "a$1" == a; do
-  if test -e "$1"; then
+  if [ -e "$1" ]; then
     unset U; unset Links
     if echo "$1"|grep "^user.0*" >/dev/null; then
       # specifying by user.0* id
       export U=$1
-      export Links=$(find . -maxdepth 1 -lname $U)
-    elif ls -l --color=none "$1"|grep ' -> ' >/dev/null; then
+      Links=$(find . -maxdepth 1 -lname "$U")
+    elif find "$1" -maxdepth 0 -type l|grep . >/dev/null; then
       # specifying by symlink
-      export Links=$1
-      export U=$(ls -l --color=none "$1"|sed -e 's/.* -> //')
+      Links=$1
+      U=$(ls -l --color=none "$1"|sed -e 's/.* -> //')
     else echo "Warning: can't make sense of username $1"; fi
-    if ! test "a$U" == a; then
-      if test -e $U/lastdate; then
+    if ! [ "a$U" == a ]; then
+      if [ -e "$U/lastdate" ]; then
         if test "a$Links" == a; then export Shortname=$U; else export Shortname=$Links; fi
-        if echo $PUBLIC_HTML | grep : >/dev/null; then
-        ssh $PUBLIC_HTML_EXTRA_SSH_OPTIONS $(echo $PUBLIC_HTML|sed -e 's/:.*//') rm -v $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$(cat $U/lastdate).*
-        else rm -v $PUBLIC_HTML/$U-$(cat $U/lastdate).*
+        if echo "$PUBLIC_HTML" | grep : >/dev/null; then
+        ssh $PUBLIC_HTML_EXTRA_SSH_OPTIONS "$(echo "$PUBLIC_HTML"|sed -e 's/:.*//')" rm -v "$(echo "$PUBLIC_HTML"|sed -e 's/[^:]*://')/$U-$(cat $U/lastdate).*"
+        else rm -v "$PUBLIC_HTML/$U-$(cat "$U/lastdate")".*
         fi
       fi
-      tar -jcvf $Shortname.tbz $U $Links
+      tar -jcvf "$Shortname.tbz" "$U" $Links
       mkdir -p old
-      mv -v --backup=numbered $Shortname.tbz old/
-      rm -rf $U $Links
+      mv -v --backup=numbered "$Shortname.tbz" old/
+      rm -rf "$U" $Links
     fi
   else echo "Warning: User $1 does not exist"; fi
 shift; done

gradint-build/server/email-lesson-report.sh → server/email-lesson-report.sh

@@ -6,7 +6,7 @@
 # - report is written to standard output so you can include
 # it in a script that makes some larger HTML page
 
-# v1.1 (C) 2007, 2009 Silas S. Brown, License: GPL
+# v1.12 (C) 2007, 2009. 2021-22 Silas S. Brown, License: GPL
 
 if ! pwd|grep email_lesson_users >/dev/null; then
   echo "This script should be run from an email_lesson_users directory (see email-lesson.sh)"
@@ -14,13 +14,13 @@ if ! pwd|grep email_lesson_users >/dev/null; then
 fi
 echo '<TABLE>'
 touch -d 0:00 /dev/shm/.midnight 2>/dev/null || touch -d 0:00 /tmp/.midnight
-if test -e /dev/shm/.midnight; then export Midnight=/dev/shm/.midnight; else export Midnight=/tmp/.midnight; fi
-for P in $(ls --color=none -t */progress.txt */podcasts-to-send 2>/dev/null); do
-  if test $P -nt $Midnight; then export Em="*";else unset Em; fi
-  if echo $P | grep podcasts-to-send$ >/dev/null; then
-    zgrep -H -m 1 . $P|grep -v ^user\.|sed -e 's,/.*:,</TD><TD COLSPAN=4>,' -e "s/^/<TR><TD>$Em/" -e "s,$,</TD></TR>,"
+if [ -e /dev/shm/.midnight ]; then Midnight=/dev/shm/.midnight; else Midnight=/tmp/.midnight; fi
+for P in $(ls --color=none -t -- */progress.txt */podcasts-to-send 2>/dev/null); do
+  if test "$P" -nt $Midnight; then Em="*";else unset Em; fi
+  if echo "$P" | grep podcasts-to-send$ >/dev/null; then
+    zgrep -H -m 1 . "$P"|grep -v ^user\.|sed -e 's,/.*:,</TD><TD COLSPAN=4>,' -e "s/^/<TR><TD>$Em/" -e "s,$,</TD></TR>,"
   else
-    zgrep -H -m 1 lessonsLeft $P|grep -v user\.|sed -e 's,/.*#,,' -e "s/^/<TR><TD>$Em/" -e "s, ,</TD><TD>,g" -e "s,$,</TD></TR>," -e "s/=/: /g"
+    zgrep -H -m 1 lessonsLeft "$P"|grep -v user\.|sed -e 's,/.*#,,' -e "s/^/<TR><TD>$Em/" -e "s, ,</TD><TD>,g" -e "s,$,</TD></TR>," -e "s/=/: /g"
   fi
 done
 rm $Midnight

gradint-build/server/email-lesson.sh → server/email-lesson.sh

@@ -3,9 +3,9 @@
 # email-lesson.sh: a script that can help you to
 # automatically distribute daily Gradint lessons
 # to students using a web server with reminder
-# emails.  Version 1.1128
+# emails.  Version 1.16
 
-# (C) 2007-2010 Silas S. Brown, License: GPL
+# (C) 2007-2010,2020-2022,2024 Silas S. Brown, License: GPL
 
 #    This program is free software; you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License as published by
@@ -17,110 +17,110 @@
 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 #    GNU General Public License for more details.
 
-export DEFAULT_SUBJECT_LINE="Vocabulary practice (automatic message from gradint)"
-export DEFAULT_FORGOT_YESTERDAY="You forgot your lesson yesterday.
+DEFAULT_SUBJECT_LINE="Vocabulary practice (automatic message from gradint)"
+DEFAULT_FORGOT_YESTERDAY="You forgot your lesson yesterday.
 Please remember to download your lesson from"
 # (NB include the words "you forgot" so that it's obvious this is a reminder not an additional lesson)
-export DEFAULT_EXPLAIN_FORGOT="Please try to hear one lesson every day.  If you download that lesson today,
+DEFAULT_EXPLAIN_FORGOT="Please try to hear one lesson every day.  If you download that lesson today,
 this program will make the next one for tomorrow."
-export DEFAULT_NEW_LESSON="Your lesson for today is at"
-export DEFAULT_LISTEN_TODAY="Please download and listen to it today."
-export DEFAULT_AUTO_MESSAGE="This is an automatic message from the gradint program.
+DEFAULT_NEW_LESSON="Your lesson for today is at"
+DEFAULT_LISTEN_TODAY="Please download and listen to it today."
+DEFAULT_AUTO_MESSAGE="This is an automatic message from the gradint program.
 Any problems, requests, or if you no longer wish to receive these emails,
 let me know."
 
-if ! test -e gradint.py; then
+if ! [ -e gradint.py ]; then
   echo "Error: This script should ALWAYS be run in the gradint directory."
   exit 1
 fi
 
-if which mail >/dev/null 2>/dev/null; then export DefaultMailProg=mail
-elif which mutt >/dev/null 2>/dev/null; then export DefaultMailProg="mutt -x"
-else export DefaultMailProg="ssh example.org mail"
+if which mail >/dev/null 2>/dev/null; then DefaultMailProg=mail
+elif which mutt >/dev/null 2>/dev/null; then DefaultMailProg="mutt -x"
+else DefaultMailProg="ssh example.org mail"
 fi
 
-if test "a$1" == "a--run"; then
+if [ "$1" == "--run" ]; then
   set -o pipefail # make sure errors in pipes are reported
-  if ! test -d email_lesson_users; then
+  if ! [ -d email_lesson_users ]; then
     echo "Error: script does not seem to have been set up yet"
     exit 1
   fi
-  export Gradint_Dir=$(pwd)
-  cd email_lesson_users
+  Gradint_Dir=$(pwd)
+  cd email_lesson_users || exit
   . config
-  if test -e $Gradint_Dir/.email-lesson-running; then
-    export Msg="Another email-lesson.sh --run is running - exitting.  (Remove $Gradint_Dir/.email-lesson-running if this isn't the case.)"
+  if [ -e "$Gradint_Dir/.email-lesson-running" ]; then
+    Msg="Another email-lesson.sh --run is running - exitting.  (Remove $Gradint_Dir/.email-lesson-running if this isn't the case.)"
     echo "$Msg"
     echo "$Msg"|$MailProg -s email-lesson-not-running $ADMIN_EMAIL # don't worry about retrying that
     exit 1
   fi
-  touch $Gradint_Dir/.email-lesson-running
-  if echo $PUBLIC_HTML | grep : >/dev/null && man ssh 2>/dev/null | grep ControlMaster >/dev/null; then
+  touch "$Gradint_Dir/.email-lesson-running"
+  if echo "$PUBLIC_HTML" | grep : >/dev/null && man ssh 2>/dev/null | grep ControlMaster >/dev/null; then
     # this version of ssh is new enough to support ControlPath, and PUBLIC_HTML indicates a remote host, so let's do it all through one connection
-    export ControlPath="-o ControlPath=$TMPDIR/__gradint_ctrl"
-    while true; do ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS -n -o ControlMaster=yes $ControlPath $(echo $PUBLIC_HTML|sed -e 's/:.*//') sleep 86400; sleep 10; done & export MasterPid=$!
+    ControlPath="-o ControlPath=$TMPDIR/__gradint_ctrl"
+    while true; do ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS -n -o ControlMaster=yes $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') sleep 86400; sleep 10; done & MasterPid=$!
   else unset MasterPid
   fi
-  (while ! bash -c "$CAT_LOGS_COMMAND"; do echo "cat-logs failed, re-trying in 61 seconds" 1>&2;sleep 61; done) | grep '/user\.' > $TMPDIR/._email_lesson_logs
+  (while ! bash -c "$CAT_LOGS_COMMAND"; do echo "cat-logs failed, re-trying in 61 seconds" >&2;sleep 61; done) | grep '/user\.' > "$TMPDIR/._email_lesson_logs"
   # (note: sleeping odd numbers of seconds so we can tell where it is if it gets stuck in one of these loops)
-  export Users="$(echo user.*)"
+  Users="$(echo user.*)"
   cd ..
   unset NeedRunMirror
   for U in $Users; do
     . email_lesson_users/config
-    if ! test "a$GLOBAL_GRADINT_OPTIONS" == a; then export GLOBAL_GRADINT_OPTIONS="$GLOBAL_GRADINT_OPTIONS ;"; fi
+    if [ "$GLOBAL_GRADINT_OPTIONS" ]; then GLOBAL_GRADINT_OPTIONS="$GLOBAL_GRADINT_OPTIONS ;"; fi
     # set some (but not all!) variables to defaults in case not set in profile
-    export SUBJECT_LINE="$DEFAULT_SUBJECT_LINE"
-    export FORGOT_YESTERDAY="$DEFAULT_FORGOT_YESTERDAY"
-    export LISTEN_TODAY="$DEFAULT_LISTEN_TODAY"
-    export NEW_LESSON="$DEFAULT_NEW_LESSON"
-    export EXPLAIN_FORGOT="$DEFAULT_EXPLAIN_FORGOT"
-    export AUTO_MESSAGE="$DEFAULT_AUTO_MESSAGE"
+    SUBJECT_LINE="$DEFAULT_SUBJECT_LINE"
+    FORGOT_YESTERDAY="$DEFAULT_FORGOT_YESTERDAY"
+    LISTEN_TODAY="$DEFAULT_LISTEN_TODAY"
+    NEW_LESSON="$DEFAULT_NEW_LESSON"
+    EXPLAIN_FORGOT="$DEFAULT_EXPLAIN_FORGOT"
+    AUTO_MESSAGE="$DEFAULT_AUTO_MESSAGE"
     unset Extra_Mailprog_Params1 Extra_Mailprog_Params2 GRADINT_OPTIONS
-    export Use_M3U=no
-    export FILE_TYPE=mp3
-    if grep $'\r' email_lesson_users/$U/profile >/dev/null; then
+    Use_M3U=no
+    FILE_TYPE=mp3
+    if grep $'\r' "email_lesson_users/$U/profile" >/dev/null; then
       # Oops, someone edited profile in a DOS line-endings editor (e.g. Wenlin on WINE for CJK stuff).  DOS line endings can mess up Extra_Mailprog_Params settings.
-      cat email_lesson_users/$U/profile | tr -d $'\r' > email_lesson_users/$U/profile.removeCR
-      mv email_lesson_users/$U/profile.removeCR email_lesson_users/$U/profile
+      tr -d $'\r' < "email_lesson_users/$U/profile" > email_lesson_users/$U/profile.removeCR
+      mv "email_lesson_users/$U/profile.removeCR" "email_lesson_users/$U/profile"
     fi
-    . email_lesson_users/$U/profile
-    if test "a$Use_M3U" == ayes; then export FILE_TYPE_2=m3u
-    else export FILE_TYPE_2=$FILE_TYPE; fi
+    . "email_lesson_users/$U/profile"
+    if [ "$Use_M3U" == yes ]; then FILE_TYPE_2=m3u
+    else FILE_TYPE_2=$FILE_TYPE; fi
     if echo "$MailProg" | grep ssh >/dev/null; then
       # ssh discards a level of quoting, so we need to be more careful
-      export SUBJECT_LINE="\"$SUBJECT_LINE\""
-      export Extra_Mailprog_Params1="\"$Extra_Mailprog_Params1\""
-      export Extra_Mailprog_Params2="\"$Extra_Mailprog_Params2\""
+      SUBJECT_LINE="\"$SUBJECT_LINE\""
+      Extra_Mailprog_Params1="\"$Extra_Mailprog_Params1\""
+      Extra_Mailprog_Params2="\"$Extra_Mailprog_Params2\""
     fi
-    if test -e email_lesson_users/$U/lastdate; then
-      if test "$(cat email_lesson_users/$U/lastdate)" == "$(date +%Y%m%d)"; then
+    if [ -e "email_lesson_users/$U/lastdate" ]; then
+      if [ "$(cat "email_lesson_users/$U/lastdate")" == "$(date +%Y%m%d)" ]; then
         # still on same day - do nothing with this user this time
 	continue
       fi
-      if ! grep $U-$(cat email_lesson_users/$U/lastdate)\. $TMPDIR/._email_lesson_logs >/dev/null
+      if ! grep "$U-$(cat email_lesson_users/$U/lastdate)"\. "$TMPDIR/._email_lesson_logs" >/dev/null
       # (don't add $FILE_TYPE after \. in case it has been changed)
       then
-        export Did_Download=0
-        if test -e email_lesson_users/$U/rollback; then
-          if test -e email_lesson_users/$U/progress.bak; then
-            mv email_lesson_users/$U/progress.bak email_lesson_users/$U/progress.txt
-            rm -f email_lesson_users/$U/progress.bin
-            export Did_Download=1 # (well actually they didn't, but we're rolling back)
+        Did_Download=0
+        if [ -e "email_lesson_users/$U/rollback" ]; then
+          if [ -e "email_lesson_users/$U/progress.bak" ]; then
+            mv "email_lesson_users/$U/progress.bak" "email_lesson_users/$U/progress.txt"
+            rm -f "email_lesson_users/$U/progress.bin"
+            Did_Download=1 # (well actually they didn't, but we're rolling back)
           fi # else can't rollback, as no progress.bak
-          if test -e email_lesson_users/$U/podcasts-to-send.old; then
-            mv email_lesson_users/$U/podcasts-to-send.old email_lesson_users/$U/podcasts-to-send
+          if [ -e "email_lesson_users/$U/podcasts-to-send.old" ]; then
+            mv "email_lesson_users/$U/podcasts-to-send.old" "email_lesson_users/$U/podcasts-to-send"
           fi
         fi
-      else export Did_Download=1; fi
-      rm -f email_lesson_users/$U/rollback
-      if test $Did_Download == 0; then
+      else Did_Download=1; fi
+      rm -f "email_lesson_users/$U/rollback"
+      if [ $Did_Download == 0 ]; then
         # send a reminder
-        export DaysOld="$(python -c "import os,time;print int((time.time()-os.stat('email_lesson_users/$U/lastdate').st_mtime)/3600/24)")"
-        if test $DaysOld -lt 5 || test $(date +%u) == 1; then # (remind only on Mondays if not checked for 5 days, to avoid filling up inboxes when people are away and can't get to email)
-        while ! $MailProg -s "$SUBJECT_LINE" $STUDENT_EMAIL "$Extra_Mailprog_Params1" "$Extra_Mailprog_Params2" <<EOF
+        DaysOld="$(python -c "import os,time;print(int((time.time()-os.stat('email_lesson_users/$U/lastdate').st_mtime)/3600/24))")"
+        if [ $DaysOld -lt 5 ] || [ $(date +%u) == 1 ]; then # (remind only on Mondays if not checked for 5 days, to avoid filling up inboxes when people are away and can't get to email)
+        while ! $MailProg -s "$SUBJECT_LINE" "$STUDENT_EMAIL" "$Extra_Mailprog_Params1" "$Extra_Mailprog_Params2" <<EOF
 $FORGOT_YESTERDAY
-$OUTSIDE_LOCATION/$U-$(cat email_lesson_users/$U/lastdate).$FILE_TYPE_2
+$OUTSIDE_LOCATION/$U-$(cat "email_lesson_users/$U/lastdate").$FILE_TYPE_2
 $EXPLAIN_FORGOT
 
 $AUTO_MESSAGE
@@ -129,68 +129,68 @@ do echo "mail sending failed; retrying in 62 seconds"; sleep 62; done; fi
         continue
       else
         # delete the previous lesson
-        if echo $PUBLIC_HTML | grep : >/dev/null; then ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo $PUBLIC_HTML|sed -e 's/:.*//') rm $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$(cat email_lesson_users/$U/lastdate).*
-        else rm $PUBLIC_HTML/$U-$(cat email_lesson_users/$U/lastdate).*; fi
+        if echo "$PUBLIC_HTML" | grep : >/dev/null; then ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') rm "$(echo "$PUBLIC_HTML"|sed -e 's/[^:]*://')/$U-$(cat "email_lesson_users/$U/lastdate").*"
+        else rm $PUBLIC_HTML/$U-$(cat "email_lesson_users/$U/lastdate").*; fi
 	# (.* because .$FILE_TYPE and possibly .m3u as well)
       fi
     fi
-    export CurDate=$(date +%Y%m%d)
-    if ! test "a$GRADINT_OPTIONS" == a; then export GRADINT_OPTIONS="$GRADINT_OPTIONS ;"; fi
-    if echo $PUBLIC_HTML | grep : >/dev/null; then export OUTDIR=$TMPDIR
-    else export OUTDIR=$PUBLIC_HTML; fi
-    export USER_GRADINT_OPTIONS="$GLOBAL_GRADINT_OPTIONS $GRADINT_OPTIONS samplesDirectory='email_lesson_users/$U/samples'; progressFile='email_lesson_users/$U/progress.txt'; pickledProgressFile='email_lesson_users/$U/progress.bin'; vocabFile='email_lesson_users/$U/vocab.txt';saveLesson='';loadLesson=0;progressFileBackup='email_lesson_users/$U/progress.bak';outputFile="
+    CurDate=$(date +%Y%m%d)
+    if [ "$GRADINT_OPTIONS" ]; then GRADINT_OPTIONS="$GRADINT_OPTIONS ;"; fi
+    if echo "$PUBLIC_HTML" | grep : >/dev/null; then OUTDIR=$TMPDIR
+    else OUTDIR=$PUBLIC_HTML; fi
+    USER_GRADINT_OPTIONS="$GLOBAL_GRADINT_OPTIONS $GRADINT_OPTIONS samplesDirectory='email_lesson_users/$U/samples'; progressFile='email_lesson_users/$U/progress.txt'; pickledProgressFile='email_lesson_users/$U/progress.bin'; vocabFile='email_lesson_users/$U/vocab.txt';saveLesson='';loadLesson=0;progressFileBackup='email_lesson_users/$U/progress.bak';outputFile="
     # (note: we DO keep progressFileBackup, because it can be useful if the server goes down and the MP3's need to be re-generated or something)
     unset Send_Podcast_Instead
-    if test -s email_lesson_users/$U/podcasts-to-send; then
-      export Send_Podcast_Instead="$(head -1 email_lesson_users/$U/podcasts-to-send)"
-      export NumLines=$(echo $(cat email_lesson_users/$U/podcasts-to-send|wc -l)-1|bc)
-      tail -$NumLines email_lesson_users/$U/podcasts-to-send > email_lesson_users/$U/podcasts-to-send2
-      mv email_lesson_users/$U/podcasts-to-send email_lesson_users/$U/podcasts-to-send.old
-      mv email_lesson_users/$U/podcasts-to-send2 email_lesson_users/$U/podcasts-to-send
-      if test $NumLines == 0; then
-        echo $U | $MailProg -s Warning:email-lesson-run-out-of-podcasts $ADMIN_EMAIL
+    if [ -s "email_lesson_users/$U/podcasts-to-send" ]; then
+      Send_Podcast_Instead="$(head -1 email_lesson_users/$U/podcasts-to-send)"
+      NumLines=$[$(cat "email_lesson_users/$U/podcasts-to-send"|wc -l)-1]
+      tail -$NumLines "email_lesson_users/$U/podcasts-to-send" > "email_lesson_users/$U/podcasts-to-send2"
+      mv "email_lesson_users/$U/podcasts-to-send" "email_lesson_users/$U/podcasts-to-send.old"
+      mv "email_lesson_users/$U/podcasts-to-send2" "email_lesson_users/$U/podcasts-to-send"
+      if [ $NumLines == 0 ]; then
+        echo "$U" | $MailProg -s Warning:email-lesson-run-out-of-podcasts $ADMIN_EMAIL
       fi
-    else rm -f email_lesson_users/$U/podcasts-to-send.old # won't be a rollback after this
+    else rm -f "email_lesson_users/$U/podcasts-to-send.old" # won't be a rollback after this
     fi
-    if test "$ENCODE_ON_REMOTE_HOST" == 1; then
-      export ToSleep=123
-      while ! if test "a$Send_Podcast_Instead" == a; then
-        python gradint.py "$USER_GRADINT_OPTIONS '-.sh'" </dev/null 2>$TMPDIR/__stderr | ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo $PUBLIC_HTML|sed -e 's/:.*//') "mkdir -p $REMOTE_WORKING_DIR; cd $REMOTE_WORKING_DIR; cat > __gradint.sh;chmod +x __gradint.sh;PATH=$SOX_PATH ./__gradint.sh|$ENCODING_COMMAND $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.$FILE_TYPE;rm -f __gradint.sh";
+    if [ "$ENCODE_ON_REMOTE_HOST" == 1 ]; then
+      ToSleep=123
+      while ! if [ ! "$Send_Podcast_Instead" ]; then
+        python gradint.py "$USER_GRADINT_OPTIONS '-.sh'" </dev/null 2>"$TMPDIR/__stderr" | ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') "mkdir -p $REMOTE_WORKING_DIR; cd $REMOTE_WORKING_DIR; cat > __gradint.sh;chmod +x __gradint.sh;PATH=$SOX_PATH ./__gradint.sh|$ENCODING_COMMAND $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.$FILE_TYPE;rm -f __gradint.sh";
       else
-        cd email_lesson_users/$U ; cat "$Send_Podcast_Instead" | ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo $PUBLIC_HTML|sed -e 's/:.*//') "cat > $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.$FILE_TYPE"; cd ../..;
+        cd "email_lesson_users/$U" ; cat "$Send_Podcast_Instead" | ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') "cat > $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.$FILE_TYPE"; cd ../..;
       fi; do
         # (</dev/null so exceptions don't get stuck on 'press enter to continue' to a temp stderr if running from a terminal)
-        $MailProg -s gradint-to-ssh-failed,-will-retry $ADMIN_EMAIL < $TMPDIR/__stderr
+        $MailProg -s gradint-to-ssh-failed,-will-retry $ADMIN_EMAIL < "$TMPDIR/__stderr"
         # (no spaces in subj so no need to decide whether to single or double quote)
         # (don't worry about mail errors - if net is totally down that's ok, admin needs to know if it's a gradint bug causing infinite loop)
-        sleep $ToSleep ; export ToSleep=$(echo $ToSleep*1.5|bc) # (increasing-time retries)
+        sleep $ToSleep ; ToSleep=$[$ToSleep*1.5] # (increasing-time retries)
       done
-      rm $TMPDIR/__stderr
-      if test "a$Use_M3U" == ayes; then
-        while ! ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo $PUBLIC_HTML|sed -e 's/:.*//') "echo $OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE > $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.m3u"; do sleep 63; done
+      rm "$TMPDIR/__stderr"
+      if [ "$Use_M3U" == yes ]; then
+        while ! ssh -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $ControlPath $(echo "$PUBLIC_HTML"|sed -e 's/:.*//') "echo $OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE > $(echo $PUBLIC_HTML|sed -e 's/[^:]*://')/$U-$CurDate.m3u"; do sleep 63; done
       fi
     else # not ENCODE_ON_REMOTE_HOST
-      if ! test "a$Send_Podcast_Instead" == a; then
-        (cd email_lesson_users/$U ; cat "$Send_Podcast_Instead") > "$OUTDIR/$U-$CurDate.$FILE_TYPE"
+      if [ "$Send_Podcast_Instead" ]; then
+        (cd "email_lesson_users/$U" ; cat "$Send_Podcast_Instead") > "$OUTDIR/$U-$CurDate.$FILE_TYPE"
       elif ! python gradint.py "$USER_GRADINT_OPTIONS '$OUTDIR/$U-$CurDate.$FILE_TYPE'" </dev/null; then
         echo "Errors from gradint itself (not ssh/network); skipping this user."
         echo "Failed on $U, check output " | $MailProg -s gradint-failed $ADMIN_EMAIL
         continue
       fi
-      if test "a$Use_M3U" == ayes; then
-        echo $OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE > $OUTDIR/$U-$CurDate.m3u
+      if [ "$Use_M3U" == yes ]; then
+        echo "$OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE" > "$OUTDIR/$U-$CurDate.m3u"
       fi
-      if echo $PUBLIC_HTML | grep : >/dev/null; then
-        while ! scp $ControlPath -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $OUTDIR/$U-$CurDate.* $PUBLIC_HTML/; do
+      if echo "$PUBLIC_HTML" | grep : >/dev/null; then
+        while ! scp $ControlPath -C $PUBLIC_HTML_EXTRA_SSH_OPTIONS $OUTDIR/$U-$CurDate.* "$PUBLIC_HTML/"; do
           echo "scp failed; re-trying in 60 seconds"
   	sleep 64
         done
-        rm $OUTDIR/$U-$CurDate.*
+        rm "$OUTDIR/$U-$CurDate".*
       fi
     fi
-    export NeedRunMirror=1
-    if ! test -e email_lesson_users/$U/progress.bak; then touch email_lesson_users/$U/progress.bak; fi # so rollback works after 1st lesson
-    while ! $MailProg -s "$SUBJECT_LINE" $STUDENT_EMAIL "$Extra_Mailprog_Params1" "$Extra_Mailprog_Params2" <<EOF
+    NeedRunMirror=1
+    if ! [ -e "email_lesson_users/$U/progress.bak" ]; then touch "email_lesson_users/$U/progress.bak"; fi # so rollback works after 1st lesson
+    while ! $MailProg -s "$SUBJECT_LINE" "$STUDENT_EMAIL" "$Extra_Mailprog_Params1" "$Extra_Mailprog_Params2" <<EOF
 $NEW_LESSON
 $OUTSIDE_LOCATION/$U-$CurDate.$FILE_TYPE_2
 $LISTEN_TODAY
@@ -198,27 +198,27 @@ $LISTEN_TODAY
 $AUTO_MESSAGE
 EOF
 do echo "mail sending failed; retrying in 65 seconds"; sleep 65; done
-    echo $CurDate > email_lesson_users/$U/lastdate
+    echo "$CurDate" > "email_lesson_users/$U/lastdate"
     unset AdminNote
-    if test "a$Send_Podcast_Instead" == a; then
-      if test $(zgrep -H -m 1 lessonsLeft email_lesson_users/$U/progress.txt|sed -e 's/.*=//') == 0; then export AdminNote="Note: $U has run out of new words"; fi
-    elif ! test -e email_lesson_users/$U/podcasts-to-send; then export AdminNote="Note: $U has run out of podcasts"; fi
-    if ! test "a$AdminNote" == a; then
-      while ! echo "$AdminNote"|$MailProg -s gradint-user-ran-out $ADMIN_EMAIL; do echo "Mail sending failed; retrying in 67 seconds"; sleep 67; done
+    if [ "$Send_Podcast_Instead" == a ]; then
+      if [ "$(zgrep -H -m 1 lessonsLeft "email_lesson_users/$U/progress.txt"|sed -e 's/.*=//')" == 0 ]; then AdminNote="Note: $U has run out of new words"; fi
+    elif ! [ -e "email_lesson_users/$U/podcasts-to-send" ]; then AdminNote="Note: $U has run out of podcasts"; fi
+    if [ "$AdminNote" ]; then
+      while ! echo "$AdminNote"|$MailProg -s gradint-user-ran-out "$ADMIN_EMAIL"; do echo "Mail sending failed; retrying in 67 seconds"; sleep 67; done
     fi
   done # end of per-user loop
-  if test "a$NeedRunMirror" == "a1" && ! test "a$PUBLIC_HTML_MIRROR_COMMAND" == a; then
+  if [ "$NeedRunMirror" == "1" ] && [ "$PUBLIC_HTML_MIRROR_COMMAND" ]; then
     while ! $PUBLIC_HTML_MIRROR_COMMAND; do
       echo "PUBLIC_HTML_MIRROR_COMMAND failed; retrying in 79 seconds"
-      echo As subject | $MailProg -s "PUBLIC_HTML_MIRROR_COMMAND failed, will retry" $ADMIN_EMAIL || true # ignore errors
+      echo As subject | $MailProg -s "PUBLIC_HTML_MIRROR_COMMAND failed, will retry" "$ADMIN_EMAIL" || true # ignore errors
       sleep 79
     done
   fi
-  rm -f $TMPDIR/._email_lesson_logs
-  if ! test a$MasterPid == a; then
+  rm -f "$TMPDIR/._email_lesson_logs"
+  if [ $MasterPid ] ; then
     kill $MasterPid
-    kill $(ps axwww|grep $TMPDIR/__gradint_ctrl|sed -e 's/^ *//' -e 's/ .*//') 2>/dev/null
-    rm -f $TMPDIR/__gradint_ctrl # in case ssh doesn't
+    kill $(pgrep -f "$TMPDIR/__gradint_ctrl") 2>/dev/null
+    rm -f "$TMPDIR/__gradint_ctrl" # in case ssh doesn't
   fi
   rm -f "$Gradint_Dir/.email-lesson-running"
   exit 0
@@ -227,10 +227,10 @@ fi
 echo "After setting up users, run this script daily with --run on the command line."
 echo "As --run was not specified, it will now go into setup mode."
 # Setup:
-if test "a$EDITOR" == a; then
+if ! [ "$EDITOR" ]; then
   echo "Error: No EDITOR environment variable set"; exit 1
 fi
-if ! test -e email_lesson_users/config; then
+if ! [ -e email_lesson_users/config ]; then
  echo "It seems the email_lesson_users directory is not set up"
  echo "Press Enter to create a new one,
  or Ctrl-C to quit if you're in the wrong directory"
@@ -238,11 +238,11 @@ if ! test -e email_lesson_users/config; then
  mkdir email_lesson_users || exit 1
  cat > email_lesson_users/config <<EOF
 # You need to edit this file.
-export GLOBAL_GRADINT_OPTIONS="" # if set, will be added to all gradint command lines (e.g. to set synthCache if it's not in advanced.txt)
-export MailProg="$DefaultMailProg" # mail, or mutt -x, or ssh some.host mail, or whatever
-export PUBLIC_HTML=~/public_html # where to put files on the WWW.  If it contains a : then scp will be used to copy them there.
-export OUTSIDE_LOCATION=http://$(hostname -f)/~$(whoami) # where they appear from outside
-export CAT_LOGS_COMMAND="false" # Please change this to a command that cats the
+GLOBAL_GRADINT_OPTIONS="" # if set, will be added to all gradint command lines (e.g. to set synthCache if it's not in advanced.txt)
+MailProg="$DefaultMailProg" # mail, or mutt -x, or ssh some.host mail, or whatever
+PUBLIC_HTML=~/public_html # where to put files on the WWW.  If it contains a : then scp will be used to copy them there.
+OUTSIDE_LOCATION=http://$(hostname -f)/~$(whoami) # where they appear from outside
+CAT_LOGS_COMMAND="false" # Please change this to a command that cats the
 # server logs for at least the last 48 hours.  (On some systems you may need
 # to make the script suid root.)  It is used to check that the users have
 # downloads their lessons and remind them if not.
@@ -254,29 +254,29 @@ export CAT_LOGS_COMMAND="false" # Please change this to a command that cats the
 # control connection (\$ControlPath will expand to
 # nothing on systems with old ssh's that don't support this)
 
-export PUBLIC_HTML_EXTRA_SSH_OPTIONS="" # if set and PUBLIC_HTML is on a remote host, these options will be added to all ssh and scp commands to that host - use this for things like specifying an alternative identity file with -i
+PUBLIC_HTML_EXTRA_SSH_OPTIONS="" # if set and PUBLIC_HTML is on a remote host, these options will be added to all ssh and scp commands to that host - use this for things like specifying an alternative identity file with -i
 
-export PUBLIC_HTML_MIRROR_COMMAND="" # if set, will be run after any new lessons are written to PUBLIC_HTML.
+PUBLIC_HTML_MIRROR_COMMAND="" # if set, will be run after any new lessons are written to PUBLIC_HTML.
 # This is for unusual setups where PUBLIC_HTML is not the real public_html directory but some command can be run to mirror its contents to the real one (perhaps on a remote server that cannot take passwordless SSH from here; of course you'd need to set up an alternative way of getting the files across and the log entries back).
 # Note: Do not add >/dev/null or similar redirects to PUBLIC_HTML_MIRROR_COMMAND as some versions of bash will give an error.
 
 export TMPDIR=/tmp # or /dev/shm or whatever
 
-export ENCODE_ON_REMOTE_HOST=0  # if 1, will ssh to the remote host
+ENCODE_ON_REMOTE_HOST=0  # if 1, will ssh to the remote host
 # that's specified in PUBLIC_HTML (which *must* be host:path in this case)
 # and will run an encoding command *there*, instead of encoding
 # locally and copying up.  This is useful if the local machine is the
 # only place gradint can run but it can't encode (e.g. Linux server running on NAS device).
 # If you set the above to 1 then you also need to set these options:
-export REMOTE_WORKING_DIR=. # directory to change to on remote host e.g. /tmp/gradint (will create with mkdir -p if does not exist)
+REMOTE_WORKING_DIR=. # directory to change to on remote host e.g. /tmp/gradint (will create with mkdir -p if does not exist)
 # (make sure $PUBLIC_HTML etc is absolute or is relative to $REMOTE_WORKING_DIR) (don't use spaces in these pathnames)
-export SOX_PATH=$PATH
+SOX_PATH=$PATH
 # make sure the above includes the remote host's "sox" as well as basic commands
-export ENCODING_COMMAND="lame --vbr-new -V 9 -"
+ENCODING_COMMAND="lame --vbr-new -V 9 -"
 # (used only if ENCODE_ON_REMOTE_HOST is set)
 # (include the full path for that if necessary; SOX_PATH will NOT be searched)
 # (set options for encode wav from stdin & output to the file specified on nxt parameter.  No shell quoting.)
-export ADMIN_EMAIL=admin@example.com # to report errors
+ADMIN_EMAIL=admin@example.com # to report errors
 EOF
  cd email_lesson_users; $EDITOR config; cd ..
  echo "Created email_lesson_users/config"
@@ -285,16 +285,16 @@ cd email_lesson_users
 while true; do
   echo "Type a user alias (or just press Enter) to add a new user, or Ctrl-C to quit"
   read Alias
-  export ID=$(mktemp -d user.$(python -c 'import random; print random.random()')XXXXXX) # (newer versions of mktemp allow more than 6 X's so the python step isn't necessary, but just in case we want to make sure that it's hard to guess the ID)
-  if ! test "a$Alias" == a; then ln -s $ID "$Alias"; fi
-  cd $ID
+  ID=$(mktemp -d user.$(python -c 'import random; print(random.random())')XXXXXX) # (newer versions of mktemp allow more than 6 X's so the python step isn't necessary, but just in case we want to make sure that it's hard to guess the ID)
+  if [ "$Alias" ]; then ln -s "$ID" "$Alias"; fi
+  cd "$ID" || exit 1
   cat > profile <<EOF
 # You need to edit the settings in this file.
-export STUDENT_EMAIL=student@example.org  # change to student's email address
+STUDENT_EMAIL=student@example.org  # change to student's email address
 export GRADINT_OPTIONS="" # extra gradint command-line options, for example to
                           # specify a different first and second language
-export FILE_TYPE=mp3 # change to something else if you want
-export Use_M3U=no # if yes, sends a .m3u link to the student
+FILE_TYPE=mp3 # change to something else if you want
+Use_M3U=no # if yes, sends a .m3u link to the student
 # instead of sending the file link directly.  Use this if
 # the student needs to stream over a slow link, but note
 # that it makes offline listening one step more complicated.
@@ -324,33 +324,33 @@ export Use_M3U=no # if yes, sends a .m3u link to the student
 # students that the lessons will now come from a different address.
 
 # Optional settings for customising the text of the message:
-export SUBJECT_LINE="$DEFAULT_SUBJECT_LINE"
-export FORGOT_YESTERDAY="$DEFAULT_FORGOT_YESTERDAY"
-export LISTEN_TODAY="$DEFAULT_LISTEN_TODAY"
-export NEW_LESSON="$DEFAULT_NEW_LESSON"
-export EXPLAIN_FORGOT="$DEFAULT_EXPLAIN_FORGOT"
-export AUTO_MESSAGE="$DEFAULT_AUTO_MESSAGE"
-export Extra_Mailprog_Params1=""
-export Extra_Mailprog_Params2=""
+SUBJECT_LINE="$DEFAULT_SUBJECT_LINE"
+FORGOT_YESTERDAY="$DEFAULT_FORGOT_YESTERDAY"
+LISTEN_TODAY="$DEFAULT_LISTEN_TODAY"
+NEW_LESSON="$DEFAULT_NEW_LESSON"
+EXPLAIN_FORGOT="$DEFAULT_EXPLAIN_FORGOT"
+AUTO_MESSAGE="$DEFAULT_AUTO_MESSAGE"
+Extra_Mailprog_Params1=""
+Extra_Mailprog_Params2=""
 # You may need to set Extra_Mailprog_Params to extra parameters
 # if the subject or text includes characters that need to be sent
-# in a specific charset.  For example, to send in Chinese GB2312
-# with Mutt, you can do this:
+# in a specific charset.  For example, to send Chinese (Simplified)
+# in UTF-8 with Mutt, you can do this:
 # export GRADINT_OPTIONS="firstLanguage='zh'; secondLanguage='en'; otherLanguages=[]"
 # export LANG=C
-# export Extra_Mailprog_Params1="-e"
-# export Extra_Mailprog_Params2="set charset='gb2312'; set send_charset='gb2312'"
-# export SUBJECT_LINE="英文词汇练习 (English vocabulary practice)"
-# export FORGOT_YESTERDAY="你忘记了昨天的课 (you forgot your lesson yesterday).
-# 请记得下载 (please remember to download) :"
-# export EXPLAIN_FORGOT="请试图天天听一课 (please try to hear one lesson every day)
-# 如果你今天下载, 这个软件要明天给你另一个课.
+# Extra_Mailprog_Params1="-e"
+# Extra_Mailprog_Params2="set charset='utf-8'; set send_charset='utf-8'"
+# SUBJECT_LINE="英文词汇练习 (English vocabulary practice)"
+# FORGOT_YESTERDAY="你忘记了昨天的课 (you forgot your lesson yesterday).
+# 请记得下载 (please remember to download) :"
+# EXPLAIN_FORGOT="请试图天天听一课 (please try to hear one lesson every day)
+# 如果你今天下载, 这个软件要明天给你另一个课.
 # (If you download that lesson today,
 # this program will make the next one for tomorrow.)"
-# export NEW_LESSON="今天的课在以下的网址 (your lesson for today is at)"
-# export LISTEN_TODAY="请你今天下载而听 (please download and listen to it today)."
-# export AUTO_MESSAGE="这个电邮是软件写的 (this is an automatic message from the gradint program).
-# 假如你有问题, 请告诉我 (any problems, let me know)."
+# NEW_LESSON="今天的课在以下的网址 (your lesson for today is at)"
+# LISTEN_TODAY="请你今天下载而听 (please download and listen to it today)."
+# AUTO_MESSAGE="这个电邮是软件写的 (this is an automatic message from the gradint program).
+# 假如你有问题, 请告诉我 (any problems, let me know)."
 
 # You can also override *some* of the email_lesson_users/config
 # options on a per-user basis by putting them here,