Merge branch 'master' into djvu_plugins

.gitignore

+*.pyc

.gitlab-ci.yml

+image: "ubuntu:18.04"
+
+before_script:
+  - apt-get update && apt-get install -y python3-nose python3-lxml python3-yaml
+
+tests:
+  script:
+    - nosetests3 -w tests/

packages/diffmerge.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<packages:packages xmlns:packages="http://www.wpkg.org/packages">
+
+	<package id="diffmerge" name="DiffMerge English" revision="%version%.1" reboot="false" priority="72">
+	<variable name="version" value="4.2.0.697" />
+
+	<check type="uninstall" condition="exists" path="SourceGear DiffMerge %version%.stable (x64)" />
+
+	<install cmd='msiexec /l* %WPKGLOGDIR%\diffmerge-%version%.log /qn /i "%WPKGSOFTWARE%\diffmerge\DiffMerge_%version%.stable_x64.msi"' />
+	<install cmd='cmd /c del /q "%Public%\Desktop\DiffMerge.lnk"' >
+		<exit code="any" />
+	</install>
+
+	<upgrade include="remove" />
+  <upgrade include="install" />
+
+	<remove cmd='msiexec /qn /x {F6BEC317-F689-4158-B1F0-F229B794CFBA}' >
+    <!-- 1605 - already removed -->
+    <exit code="1605" />
+  </remove>
+
+	</package>
+
+ </packages:packages>

packages/djvu_desktop.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<packages:packages xmlns:packages="http://www.wpkg.org/packages">
+
+  <package id="djvu_desktop" name="DjVu desktop" revision="%version%.1" reboot="false" priority="91">
+	<variable name="version" value="2.1" />
+  <check type="uninstall" condition="exists" path="WinDjView %version%" />
+
+	<install cmd='cmd /c "%WPKGSOFTWARE%\djvu\WinDjView-%version%-Setup.exe" /S' />
+
+  <upgrade include="remove" />
+  <upgrade include="install" />
+
+	<remove cmd='cmd /c %WPKGSOFTWARE%\wpkg\tools\waitforprocess.cmd Au_.exe' />
+	<remove cmd='"%PROGRAMFILES%\WinDjView\uninstall.exe" /S' />
+	<remove cmd='cmd /c %WPKGSOFTWARE%\wpkg\tools\waitforprocess.cmd Au_.exe' />
+
+  </package>
+
+</packages:packages>

tests/config.yml

+---
+package_dir: ../packages
+
+package_glob: ../packages/*.xml
+
+# tests_to_skip is a dict of lists. The key is the name
+# of a test to skip, and then with with a value which is
+# the list of filenames for which the test should not
+# be run. The filename should be relative to package_dir
+#
+# Example: suppose you want to skip the test test_example
+#          for files 7zip.xml and winscp.xml in package_dir
+#
+# tests_to_skip:
+#  test_example:
+#  - 7zip.xml
+#  - winscp.xml
+
+variable_names_to_skip:
+  openssh_srv.xml:
+    - wpkgwininfo
+
+# This should be a suitable subset of the default env vars
+# expected in the ucam_wpkg repo.
+repo_var_names:
+  - WPKGSOFTWARE
+  - WPKGINSTITUTION
+  - WPKGSHAREBASE
+  - WPKGLOGDIR
+
+# default environment variables that Windows defines
+win_var_names:
+  - programfiles(x86)
+  - programfiles
+  - programdata
+  - allusersprofile
+  - comspec
+  - computername
+  - windir
+  - systemdrive
+  - programfileswpkg
+  - temp
+  - systemroot
+  - public
+  - arch
+

tests/test_var_names.py

+import glob
+import os
+import re
+import lxml, lxml.etree
+from util import *
+
+config = Config().config
+xmllist = XMLList(config)
+
+default_var_names = config['repo_var_names'] + config['win_var_names']
+
+variable_re = re.compile('%(.*?)%')
+
+def test_var_names():
+    for file_name in xmllist.getfiles('test_var_names'):
+        with open(file_name, 'r') as file:
+            data = file.read().replace('\n', '')    
+
+        lc_vars = [x.lower() for x in variable_re.findall(data)]
+        doc = lxml.etree.parse(file_name)
+        pkg_vars = doc.xpath('//variable')
+        allowed_pkg_vars = [x.get('name') for x in pkg_vars]
+
+        skipped_pkg_vars = config.get('variable_names_to_skip').get(os.path.basename(file_name), [])
+
+        for var in set(lc_vars):
+            yield var_name_is_valid, file_name, var
+            yield var_name_is_allowed, var, default_var_names + allowed_pkg_vars + skipped_pkg_vars, file_name
+
+def var_name_is_valid(filename, varname): 
+    # NB there are very few restrictions on var names imposed by Windows.
+    # But for sanity, we restrict ourselves to what we consider to be a
+    # "sensible" subset
+    valid_name_re = re.compile('^[a-z0-9()_-]*$')
+    assert valid_name_re.match(varname)
+
+def var_name_is_allowed(varname, allowed_vars, file_name):
+    assert varname in [x.lower() for x in allowed_vars]

tests/test_xml_validates.py

+import glob
+from util import *
+
+xmllist = XMLList(Config().config)
+
+validator = XSDValidator('../xsd/packages.xsd')
+
+def test_xsd():
+    for file_name in xmllist.getfiles('test_xsd'):
+        yield check_file, file_name 
+
+def check_file(file_name):
+      assert validator.validate(file_name)

tests/util.py

+from lxml import etree 
+from lxml.etree import DocumentInvalid
+import lxml
+import glob
+import os
+import yaml
+
+# import pprint
+# pp = pprint.PrettyPrinter(indent=2)
+
+class XSDValidator:
+
+    def __init__(self, xsd_path):
+        xmlschema_doc = etree.parse(xsd_path)
+        self.xmlschema = etree.XMLSchema(xmlschema_doc)
+
+    def validate(self, xml_path):
+        xml_doc = etree.parse(xml_path)
+        try:
+            self.xmlschema.assertValid(xml_doc)
+        except lxml.etree.DocumentInvalid:
+            print("Validation error(s):")
+            for error in self.xmlschema.error_log:
+                print("  Line {}: {}".format(error.line, error.message))
+        return self.xmlschema.validate(xml_doc)
+
+
+class XMLList:
+
+    def __init__(self, config):
+        self.config = config
+        self.files = set(glob.glob(self.config['package_glob']))
+
+    def getfiles(self, testname):
+        if 'tests_to_skip' in self.config.keys() and testname in self.config['tests_to_skip'].keys():
+            files_to_skip = { os.path.join(self.config['package_dir'], x) for x in self.config['tests_to_skip'][testname] or []}
+            return self.files - files_to_skip
+        else:
+            return self.files
+
+class Config:
+
+    def __init__(self):
+        with open('config.yml', 'r') as conffile:
+            self.defaults = {}
+            self.defaults['variable_names_to_skip'] = {}
+
+            config = yaml.load(conffile)
+            self.config = {**self.defaults, **config}

xsd/config.xsd

+<?xml version="1.0" encoding="UTF-8"?>
+<xsd:schema targetNamespace="http://www.wpkg.org/config" elementFormDefault="qualified"
+	xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns="http://www.wpkg.org/config">
+
+	<xsd:complexType name="config">
+		<xsd:sequence minOccurs="1" maxOccurs="1">
+			<xsd:sequence minOccurs="0" maxOccurs="1">
+				<xsd:element name="languages" type="languages"></xsd:element>
+			</xsd:sequence>
+			<xsd:choice minOccurs="0" maxOccurs="unbounded">
+				<xsd:element name="param" type="xsd:string"></xsd:element>
+			</xsd:choice>
+		</xsd:sequence>
+	</xsd:complexType>
+
+	<xsd:complexType name="param">
+		<xsd:annotation>
+			<xsd:documentation>Configuration parameter entry. Defines a configuration parameter of WPKG.</xsd:documentation>
+		</xsd:annotation>
+		<xsd:sequence></xsd:sequence>
+		<xsd:attribute name="name" type="xsd:string" use="required">
+			<xsd:annotation>
+				<xsd:documentation>Name of the configuration parameter.</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+		<xsd:attribute name="value" type="xsd:string" use="required">
+			<xsd:annotation>
+				<xsd:documentation>Value to be assigned to the configuration parameter.</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+	</xsd:complexType>
+
+	<xsd:complexType name="languages">
+		<xsd:annotation>
+			<xsd:documentation>Holds language definitions such as language-specific strings.</xsd:documentation>
+		</xsd:annotation>
+		<xsd:choice minOccurs="0" maxOccurs="unbounded">
+			<xsd:element name="language" type="language"></xsd:element>
+		</xsd:choice>
+	</xsd:complexType>
+
+	<xsd:complexType name="language">
+		<xsd:annotation>
+			<xsd:documentation>Language holds a concrete language specification defining message strings.</xsd:documentation>
+		</xsd:annotation>
+		<xsd:choice minOccurs="0" maxOccurs="unbounded">
+			<xsd:element name="string" type="string"></xsd:element>
+		</xsd:choice>
+		<xsd:attribute name="lcid">
+			<xsd:annotation>
+				<xsd:documentation>
+					Comma-separated list of LCID values for which this language specification is valid. For example
+					specify "409,809" to match English locale. For a complete list of LCIDs see
+					http://www.microsoft.com/globaldev/reference/lcid-all.mspx.
+				</xsd:documentation>
+			</xsd:annotation>
+			<xsd:simpleType>
+				<xsd:restriction base="xsd:string">
+					<xsd:pattern value="[\w]+(,[\w]+)*"></xsd:pattern>
+				</xsd:restriction>
+			</xsd:simpleType>
+		</xsd:attribute>
+	</xsd:complexType>
+
+	<xsd:complexType name="string" mixed="true">
+		<xsd:annotation>
+			<xsd:documentation>Localized string definition.</xsd:documentation>
+		</xsd:annotation>
+		<xsd:attribute name="id">
+			<xsd:annotation>
+				<xsd:documentation>
+					Identifier for language string. Identifies the string you are going to localize.
+				</xsd:documentation>
+			</xsd:annotation>
+			<xsd:simpleType>
+				<xsd:restriction base="xsd:string">
+					<xsd:enumeration value="notifyUserStart"></xsd:enumeration>
+					<xsd:enumeration value="notifyUserStop"></xsd:enumeration>
+					<xsd:enumeration value="notifyUserFail"></xsd:enumeration>
+					<xsd:enumeration value="notifyUserReboot"></xsd:enumeration>
+				</xsd:restriction>
+			</xsd:simpleType>
+		</xsd:attribute>
+	</xsd:complexType>
+
+    <xsd:element name="config" type="config">
+    	<xsd:annotation>
+    		<xsd:documentation>Root element of WPKG configuration.</xsd:documentation>
+    	</xsd:annotation></xsd:element>
+</xsd:schema>
\ No newline at end of file

xsd/hosts.xsd

+<?xml version="1.0" encoding="UTF-8"?>
+<xsd:schema targetNamespace="http://www.wpkg.org/hosts"
+	elementFormDefault="unqualified" xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+	xmlns="http://www.wpkg.org/hosts" xmlns:wpkg="http://www.wpkg.org/wpkg">
+
+	<xsd:import namespace="http://www.wpkg.org/wpkg" schemaLocation="wpkg.xsd">
+		<xsd:annotation>
+			<xsd:documentation>Include WPKG master definition.
+			</xsd:documentation>
+		</xsd:annotation>
+	</xsd:import>
+
+	<xsd:element name="wpkg" type="wpkg">
+		<xsd:annotation>
+			<xsd:documentation>Root element for host definitions.
+			</xsd:documentation>
+		</xsd:annotation>
+	</xsd:element>
+
+	<xsd:complexType name="wpkg">
+		<xsd:sequence minOccurs="1" maxOccurs="unbounded">
+			<xsd:element name="host" type="host"></xsd:element>
+		</xsd:sequence>
+	</xsd:complexType>
+
+	<xsd:complexType name="host">
+		<xsd:sequence maxOccurs="1" minOccurs="0">
+			<xsd:sequence maxOccurs="1" minOccurs="0">
+				<xsd:element name="condition" type="wpkg:condition">
+					<xsd:annotation>
+						<xsd:documentation>Define condition under which this host entry is applied.</xsd:documentation>
+					</xsd:annotation></xsd:element>
+			</xsd:sequence>
+			<xsd:choice maxOccurs="unbounded" minOccurs="0">
+				<xsd:element name="variable" type="wpkg:variable">
+					<xsd:annotation>
+						<xsd:documentation>Define an environment variable to be set when the host entry applies.</xsd:documentation>
+					</xsd:annotation></xsd:element>
+				<xsd:element name="profile" type="profileReference">
+					<xsd:annotation>
+						<xsd:documentation>Define profile which should be applied to matching host.</xsd:documentation>
+					</xsd:annotation></xsd:element>
+			</xsd:choice>
+		</xsd:sequence>
+		<xsd:attribute name="name" type="xsd:string" use="optional">
+			<xsd:annotation>
+				<xsd:documentation>Hostname specification. It might contain regular expressions as well as well 
+as IP-address ranges.
+
+Direct match: This is tried first always. If the hostname matches exactly the 
+value of 'name' this host node is applied to the machine.
+
+IP-Ranges: format has to be specified as follows: 
+start[-end].start[-end].start[-end].start[-end] examples: 192.168.1.1 
+192.168.1.1-254 192.168.1-5.20-50
+
+Regular expressions: example: &quot;test-.*&quot; will match all machines where the 
+hostname is starting with &quot;test-&quot; string.
+
+If no name attribute exists then the host entry will always match. You can 
+omit the name attribute and use extended attribute matching like hostname 
+(which matches the host name only, but not IP), ipaddresses, domainname etc.
+</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+		<xsd:attribute name="profile-id" type="xsd:string"
+			use="optional">
+			<xsd:annotation>
+				<xsd:documentation>Main profile. You can specify the applying profile as separate &quot;profile&quot; nodes 
+as well but if there is only one single profile it is recommended to use the 
+profile-id attribute.
+</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+		<xsd:attributeGroup ref="wpkg:HostMatch"></xsd:attributeGroup>
+	</xsd:complexType>
+
+	<xsd:complexType name="profileReference">
+		<xsd:sequence maxOccurs="1" minOccurs="0">
+			<xsd:element name="condition" type="wpkg:condition"></xsd:element>
+		</xsd:sequence>
+		<xsd:attribute name="id" type="xsd:string" use="required">
+			<xsd:annotation>
+				<xsd:documentation>
+					ID referencing to the profile defined in
+					profiles.xml
+				</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+		<xsd:attributeGroup ref="wpkg:HostMatch"></xsd:attributeGroup>
+	</xsd:complexType>
+</xsd:schema>
\ No newline at end of file

xsd/profiles.xsd

+<?xml version="1.0" encoding="UTF-8"?>
+<xsd:schema targetNamespace="http://www.wpkg.org/profiles"
+	elementFormDefault="unqualified" xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+	xmlns="http://www.wpkg.org/profiles" xmlns:wpkg="http://www.wpkg.org/wpkg">
+
+	<xsd:import namespace="http://www.wpkg.org/wpkg" schemaLocation="wpkg.xsd">
+		<xsd:annotation>
+			<xsd:documentation>Include WPKG master definition.</xsd:documentation>
+		</xsd:annotation>
+	</xsd:import>
+
+	<xsd:element name="profiles" type="profiles">
+		<xsd:annotation>
+			<xsd:documentation>Root element for profile definitions.
+			</xsd:documentation>
+		</xsd:annotation>
+	</xsd:element>
+
+	<xsd:complexType name="profiles">
+		<xsd:sequence maxOccurs="unbounded" minOccurs="1">
+			<xsd:element name="profile" type="profile"></xsd:element>
+		</xsd:sequence>
+	</xsd:complexType>
+
+	<xsd:complexType name="profile">
+		<xsd:annotation>
+			<xsd:documentation></xsd:documentation>
+		</xsd:annotation>
+		<xsd:choice minOccurs="0" maxOccurs="unbounded">
+			<xsd:element name="depends" type="depends"></xsd:element>
+			<xsd:element name="variable" type="wpkg:variable"></xsd:element>
+			<xsd:element name="package" type="packageReference"></xsd:element>
+		</xsd:choice>
+		<xsd:attribute name="id" type="xsd:string" use="required">
+			<xsd:annotation>
+				<xsd:documentation>
+Name of the profile (as referenced within hosts.xml).
+				</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+	</xsd:complexType>
+
+	<xsd:complexType name="packageReference">
+		<xsd:complexContent>
+			<xsd:extension base="wpkg:packageReference">
+				<xsd:attribute name="installdate" type="xsd:dateTime"
+					use="optional">
+					<xsd:annotation>
+						<xsd:documentation>
+Date from which the package should be installed (this date or later). Date has to
+be specified in ISO 8601 format (see http://www.w3.org/TR/NOTE-datetime).
+						</xsd:documentation>
+					</xsd:annotation>
+				</xsd:attribute>
+				<xsd:attribute name="uninstalldate" type="xsd:dateTime"
+					use="optional">
+					<xsd:annotation>
+						<xsd:documentation>
+Date from which the package should be removed (this date or later). Date has to be
+specified in ISO 8601 format (seehttp://www.w3.org/TR/NOTE-datetime).
+						</xsd:documentation>
+					</xsd:annotation>
+				</xsd:attribute>
+			</xsd:extension>
+		</xsd:complexContent>
+	</xsd:complexType>
+
+	<xsd:complexType name="depends">
+		<xsd:sequence minOccurs="0" maxOccurs="1">
+			<xsd:element name="condition" type="wpkg:condition"></xsd:element>
+		</xsd:sequence>
+		<xsd:attribute name="profile-id" type="xsd:string"
+			use="required">
+			<xsd:annotation>
+				<xsd:documentation>
+ID of the profile which this profile depends on (in
+fact it works similar to an include - all packages
+from the referenced profile are installed too).
+				</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+		<xsd:attributeGroup ref="wpkg:HostMatch"></xsd:attributeGroup>
+	</xsd:complexType>
+</xsd:schema>
\ No newline at end of file

xsd/settings.xsd

+<?xml version="1.0" encoding="UTF-8"?>
+<xsd:schema targetNamespace="http://www.wpkg.org/settings"
+	elementFormDefault="unqualified"
+	xmlns:xsd="http://www.w3.org/2001/XMLSchema"
+	xmlns="http://www.wpkg.org/settings"
+	xmlns:packages="http://www.wpkg.org/packages" xmlns:wpkg="http://www.wpkg.org/wpkg">
+
+	<xsd:import namespace="http://www.wpkg.org/packages" schemaLocation="packages.xsd">
+		<xsd:annotation>
+			<xsd:documentation>
+				Include WPKG package definition.
+			</xsd:documentation>
+		</xsd:annotation>
+	</xsd:import>
+
+	<xsd:import namespace="http://www.wpkg.org/wpkg" schemaLocation="wpkg.xsd">
+		<xsd:annotation>
+			<xsd:documentation>
+				Include WPKG generic elements.
+			</xsd:documentation>
+		</xsd:annotation>
+	</xsd:import>
+
+
+	<xsd:element name="wpkg" type="wpkg">
+		<xsd:annotation>
+			<xsd:documentation>Root element for local settings database.</xsd:documentation>
+		</xsd:annotation></xsd:element>
+
+	<xsd:complexType name="wpkg">
+		<xsd:annotation>
+			<xsd:documentation>
+				Type of loca settings database. Contains all packages
+				installed on local node.
+			</xsd:documentation>
+		</xsd:annotation>
+		<xsd:choice minOccurs="1" maxOccurs="unbounded">
+			<xsd:sequence minOccurs="0" maxOccurs="1"><xsd:element name="checkResults" type="checkResults">
+	<xsd:annotation>
+		<xsd:documentation>List of executed checks and their results.</xsd:documentation>
+	</xsd:annotation></xsd:element></xsd:sequence>
+			<xsd:choice minOccurs="0" maxOccurs="unbounded">
+				<xsd:annotation>
+					<xsd:documentation>
+						References to packages which are installed on
+						the host.
+					</xsd:documentation>
+				</xsd:annotation>
+				<xsd:element name="package" type="packages:package">
+					<xsd:annotation>
+						<xsd:documentation>Package installed on node.</xsd:documentation>
+					</xsd:annotation></xsd:element>
+			</xsd:choice>
+		</xsd:choice>
+		<xsd:attribute name="hostname" type="xsd:string">
+			<xsd:annotation>
+				<xsd:documentation>
+					Name of host where the settings have been generated.
+				</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+		<xsd:attribute name="architecture" type="xsd:string">
+			<xsd:annotation>
+				<xsd:documentation>
+					Architecture of host where the settings have been
+					generated.
+				</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+		<xsd:attribute name="os" type="xsd:string">
+			<xsd:annotation>
+				<xsd:documentation>
+					Operating system on host where the settings have
+					been generated.
+				</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+		<xsd:attribute name="ipaddresses" type="xsd:string">
+			<xsd:annotation>
+				<xsd:documentation>
+					List of IP addresses on host where the settings have
+					been generated.
+				</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+		<xsd:attribute name="domainname" type="xsd:string">
+			<xsd:annotation>
+				<xsd:documentation>
+					Name of domain on host where the settings have been
+					generated.
+				</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+		<xsd:attribute name="groups" type="xsd:string">
+			<xsd:annotation>
+				<xsd:documentation>
+					Name of primary group on host where the settings
+					have been generated.
+				</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+		<xsd:attribute name="lcid" type="xsd:string">
+			<xsd:annotation>
+				<xsd:documentation>
+					Locale setting (LCID) of user under which WPKG has
+					been executed when the settings have been generated.
+				</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+		<xsd:attribute name="lcidOS" type="xsd:string">
+			<xsd:annotation>
+				<xsd:documentation>
+					Locale setting (LCID) of operating system on which
+					WPKG has been executed when the settings have been
+					generated.
+				</xsd:documentation>
+			</xsd:annotation>
+		</xsd:attribute>
+	</xsd:complexType>
+
+	<xsd:complexType name="checkResults">
+        <xsd:annotation>
+        	<xsd:documentation>Stores results of checks executed on node.
+This can be used on remote host in order to know the evaluated result of a specific check on a remote client.
+It might also be useful for debugging to verify the results of specific checks.</xsd:documentation>
+        </xsd:annotation>
+        <xsd:choice minOccurs="0" maxOccurs="unbounded">
+			<xsd:element name="check" type="check">
+				<xsd:annotation>
+					<xsd:documentation>Refers to a check and stores the check result.</xsd:documentation>
+				</xsd:annotation></xsd:element>
+		</xsd:choice>
+	</xsd:complexType>
+
+	<xsd:complexType name="check">
+		<xsd:complexContent>
+			<xsd:extension base="wpkg:check">
+				<xsd:attribute name="result" type="xsd:boolean"
+					use="required">
+					<xsd:annotation>
+						<xsd:documentation>
+							The check result is only used to store the
+							result in local settings database in order
+							to remember the check result on a remote
+							system.
+						</xsd:documentation>
+					</xsd:annotation>
+				</xsd:attribute>
+			</xsd:extension>
+		</xsd:complexContent>
+	</xsd:complexType>
+</xsd:schema>
\ No newline at end of file