completed elimination of jaxml

app/scodoc/sco_utils.py

@@ -37,24 +37,16 @@ from hashlib import md5
 import numbers
 import os
 import re
-import sys
+import six
 import six.moves._thread
+import sys
 import time
 import types
 import six.moves.urllib.request, six.moves.urllib.parse, six.moves.urllib.error
 import six.moves.urllib.request, six.moves.urllib.error, six.moves.urllib.parse
-
+from xml.etree.ElementTree import Element
-
-# XML generation package (apt-get install jaxml)
-import jaxml  # XXX
-
-try:
-    import six
 
 STRING_TYPES = six.string_types
-except ImportError:
-    # fallback for very old ScoDoc instances
-    STRING_TYPES = bytes
 
 from PIL import Image as PILImage
 
@@ -876,9 +868,8 @@ def _sco_error_response(context, msg, format="html", REQUEST=None):
         raise sco_exceptions.ScoValueError(msg)
     elif format == "xml":
         REQUEST.RESPONSE.setHeader("content-type", XML_MIMETYPE)
-        doc = jaxml.XML_document(encoding=SCO_ENCODING)
+        doc = ElementTree.Element("error", msg=msg)
-        doc.error(msg=msg)
+        return sco_xml.XML_HEADER + ElementTree.tostring(doc)
-        return repr(doc)
     elif format == "json":
         REQUEST.RESPONSE.setHeader("content-type", JSON_MIMETYPE)
         return "undefined"  # XXX voir quoi faire en cas d'erreur json

app/views/absences.py

@@ -46,16 +46,16 @@ L'API de plus bas niveau est en gros:
 
 """
 
-import string
+import calendar
-import re
+import cgi
-import time
 import datetime
 import dateutil
 import dateutil.parser
-import calendar
+import re
 import six.moves.urllib.request, six.moves.urllib.parse, six.moves.urllib.error
-import cgi
+import string
-import jaxml
+import time
+from xml.etree import ElementTree
 
 from flask import g
 from flask import current_app
@@ -93,6 +93,7 @@ from app.scodoc import sco_groups
 from app.scodoc import sco_groups_view
 from app.scodoc import sco_moduleimpl
 from app.scodoc import sco_preferences
+from app.scodoc import sco_xml
 
 
 CSSSTYLES = html_sco_header.BOOTSTRAP_MULTISELECT_CSS
@@ -1505,22 +1506,22 @@ def XMLgetAbsEtud(context, beg_date="", end_date="", REQUEST=None):
     Abs = sco_abs.ListeAbsDate(context, etud["etudid"], beg_date, end_date)
 
     REQUEST.RESPONSE.setHeader("content-type", scu.XML_MIMETYPE)
-    doc = jaxml.XML_document(encoding=scu.SCO_ENCODING)
+    doc = ElementTree.Element(
-    doc.absences(etudid=etud["etudid"], beg_date=beg_date, end_date=end_date)
+        "absences", etudid=etud["etudid"], beg_date=beg_date, end_date=end_date
-    doc._push()
+    )
     for a in Abs:
         if a["estabs"]:  # ne donne pas les justifications si pas d'absence
-            doc._push()
+            doc.append(
-            doc.abs(
+                ElementTree.Element(
+                    "abs",
                     begin=a["begin"],
                     end=a["end"],
                     description=a["description"],
                     justified=a["estjust"],
                 )
-            doc._pop()
+            )
-    doc._pop()
     log("XMLgetAbsEtud (%gs)" % (time.time() - t0))
-    return repr(doc)
+    return sco_xml.XML_HEADER + ElementTree.tostring(doc)
 
 
 context.populate(globals())

app/views/notes.py

@@ -33,9 +33,9 @@ Emmanuel Viennet, 2021
 import sys
 import time
 import datetime
-import jaxml
 import pprint
 from operator import itemgetter
+from xml.etree import ElementTree
 
 from flask import url_for, g
 from flask import current_app
@@ -128,6 +128,7 @@ from app.scodoc import sco_tag_module
 from app.scodoc import sco_ue_external
 from app.scodoc import sco_undo_notes
 from app.scodoc import sco_users
+from app.scodoc import sco_xml
 from app.scodoc.gen_tables import GenTable
 from app.scodoc.sco_pdf import PDFLOCK
 from app.scodoc.sco_permissions import Permission
@@ -650,13 +651,11 @@ def XMLgetFormsemestres(context, etape_apo=None, formsemestre_id=None, REQUEST=N
         args["formsemestre_id"] = formsemestre_id
     if REQUEST:
         REQUEST.RESPONSE.setHeader("content-type", scu.XML_MIMETYPE)
-    doc = jaxml.XML_document(encoding=scu.SCO_ENCODING)
+    doc = ElementTree.Element("formsemestrelist")
-    doc.formsemestrelist()
     for sem in sco_formsemestre.do_formsemestre_list(context, args=args):
-        doc._push()
+        doc.append("formsemestre", **sem)
-        doc.formsemestre(sem)
+
-        doc._pop()
+    return sco_xml.XML_HEADER + ElementTree.tostring(doc)
-    return repr(doc)
 
 
 sco_publish(

app/views/users.py

@@ -34,7 +34,7 @@ Vues s'appuyant sur auth et sco_users
 Emmanuel Viennet, 2021
 """
 import re
-import jaxml
+from xml.etree import ElementTree
 
 from flask import g
 from flask_login import current_user
@@ -54,6 +54,7 @@ from app.decorators import (
 from app.scodoc import html_sco_header
 from app.scodoc import sco_users
 from app.scodoc import sco_utils as scu
+from app.scodoc import sco_xml
 from app.scodoc.notes_log import log
 from app.scodoc.sco_exceptions import AccessDenied, ScoValueError
 from app.scodoc.sco_permissions_check import can_handle_passwd
@@ -341,7 +342,7 @@ def create_user_form(context, REQUEST, user_name=None, edit=0):
             edit = 0
         try:
             force = int(vals["force"][0])
-        except:
+        except (ValueError, TypeError):
             force = 0
 
         if edit:
@@ -471,13 +472,12 @@ def get_user_list_xml(dept=None, start="", limit=25, REQUEST=None):
     ]
     if REQUEST:
         REQUEST.RESPONSE.setHeader("content-type", scu.XML_MIMETYPE)
-    doc = jaxml.XML_document(encoding=scu.SCO_ENCODING)
+    doc = ElementTree.Element("results")
-    doc.results()
     for user in userlist[:limit]:
-        doc._push()
+        x_rs = ElementTree.Element("rs", id=user.id, info="")
-        doc.rs(user.get_nomplogin(), id=user.id, info="")
+        x_rs.text = user.get_nomplogin()
-        doc._pop()
+        doc.append(x_rs)
-    return repr(doc)
+    return sco_xml.XML_HEADER + ElementTree.tostring(doc)
 
 
 @bp.route("/form_change_password")

config/install_debian10.sh

@@ -87,9 +87,8 @@ apt-get -y install postgresql
 apt-get -y install graphviz
 
 # ------------ INSTALL DES EXTENSIONS PYTHON (2.7)
-# XXX to fix: pip in our env
+# ScoDoc8 uses pip in our env
 apt-get -y install python-docutils
-apt-get -y install python-jaxml 
 apt-get -y install python-psycopg2 
 apt-get -y install python-pyrss2gen 
 apt-get -y install python-pil python-reportlab 

misc/iscid_create_formation_from_xls.py

@@ -7,11 +7,12 @@
 
 import os, sys, pdb, pprint
 from openpyxl import load_workbook  # apt-get install python-openpyxl
-import jaxml
+from xml.etree import ElementTree
-SCO_ENCODING = 'utf-8'
+
+SCO_ENCODING = "utf-8"
 
 INPUT_FILENAME = "/tmp/Bachelor.xlsx"
-OUTPUT_FILENAME= os.path.splitext(INPUT_FILENAME)[0] + '.xml' 
+OUTPUT_FILENAME = os.path.splitext(INPUT_FILENAME)[0] + ".xml"
 
 FIRST_SHEET_IDX = 1  # saute première feuille du classeur
 
@@ -24,7 +25,7 @@ UE_ELECTIVE = 4 # UE "élective" dans certains parcours (UCAC?, ISCID)
 UE_PROFESSIONNELLE = 5  # UE "professionnelle" (ISCID, ...)
 
 # Code du fichier Excel:
-UE_TYPE2CODE = { u'UE F' : UE_STANDARD, u'UE E' : UE_ELECTIVE }
+UE_TYPE2CODE = {u"UE F": UE_STANDARD, u"UE E": UE_ELECTIVE}
 
 # Lecture du fichier Excel
 UE = []
@@ -32,11 +33,11 @@ wb = load_workbook(filename=INPUT_FILENAME)
 # print wb.get_sheet_names()
 
 for sheet_name in wb.get_sheet_names()[FIRST_SHEET_IDX:]:
-    print 'Importing sheet %s' % sheet_name
+    print "Importing sheet %s" % sheet_name
     sheet = wb.get_sheet_by_name(sheet_name)
     # Avance jusqu'à trouver le titre 'CODE' en premiere colonne
     i = 0
-    while i < len(sheet.rows) and sheet.rows[i][0].value != 'CODE':
+    while i < len(sheet.rows) and sheet.rows[i][0].value != "CODE":
         i = i + 1
 
     i = i + 1
@@ -53,28 +54,31 @@ for sheet_name in wb.get_sheet_names()[FIRST_SHEET_IDX:]:
                 acronyme = sheet.rows[i + 1][0].value  # code module sur ligne suivante
                 # print acronyme
                 if acronyme:  # tres specifique: deduit l'acronyme d'UE du code module
-                    parts = acronyme.split(u'-')
+                    parts = acronyme.split(u"-")
                     parts[-1] = parts[-1][-1]  # ne garde que le dernier chiffre
-                    acronyme = u'-'.join(parts) # B1-LV1-EN1 -> B1-LV1-1
+                    acronyme = u"-".join(parts)  # B1-LV1-EN1 -> B1-LV1-1
                 # print '->', acronyme
             if not acronyme:
                 acronyme = sheet.rows[i][3].value  # fallback: titre
-            ue = { 'acronyme' : acronyme,
+            ue = {
-                   'titre' : sheet.rows[i][3].value,
+                "acronyme": acronyme,
-                   'ects' : sheet.rows[i][5].value or u"",
+                "titre": sheet.rows[i][3].value,
-                   'type' : UE_TYPE2CODE[type_ue],
+                "ects": sheet.rows[i][5].value or u"",
-                   'numero' : (sheet.rows[i][1].value or 0)*1000 + i*10,
+                "type": UE_TYPE2CODE[type_ue],
-                   'modules' : []
+                "numero": (sheet.rows[i][1].value or 0) * 1000 + i * 10,
+                "modules": [],
             }
             i_ue = i
         if code:
-            ue['modules'].append( {
+            ue["modules"].append(
-                'code' : code,
+                {
-                'heures_td' : sheet.rows[i_ue][4].value or u"",
+                    "code": code,
-                'titre' : sheet.rows[i][3].value,
+                    "heures_td": sheet.rows[i_ue][4].value or u"",
-                'semestre_id' : sheet.rows[i][1].value,
+                    "titre": sheet.rows[i][3].value,
-                'numero' : i*10
+                    "semestre_id": sheet.rows[i][1].value,
-                } )
+                    "numero": i * 10,
+                }
+            )
 
         i += 1  # next line
 
@@ -83,46 +87,55 @@ for sheet_name in wb.get_sheet_names()[FIRST_SHEET_IDX:]:
 
 
 def sstr(s):
-    if type(s) is type(u''):
+    if type(s) is type(u""):
         return s.encode(SCO_ENCODING)
     else:
         return str(s)
 
-# ----- Write to XML    
-doc = jaxml.XML_document( encoding=SCO_ENCODING )
 
-doc._push()
+# ----- Write to XML
-doc.formation( acronyme="Bachelor ISCID",
+doc = ElementTree.Element(
+    "formation",
+    acronyme="Bachelor ISCID",
     code_specialite="",
     type_parcours="1001",
     titre_officiel="Bachelor ISCID",
     formation_code="FCOD4",
     version="1",
     titre="Bachelor ISCID",
-               formation_id="FORM115"
+    formation_id="FORM115",
 )
 
 for ue in UE:
-    doc._push()
+    x_ue = ElementTree.Element(
-    doc.ue( acronyme=sstr(ue['acronyme']), ects=sstr(ue['ects']), titre=sstr(ue['titre']), numero=sstr(ue['numero']), type=sstr(ue['type']) )
+        "ue",
-    doc._push()
+        acronyme=sstr(ue["acronyme"]),
-    doc.matiere( titre=sstr(ue['titre']) ) # useless but necessary
+        ects=sstr(ue["ects"]),
-    for m in ue['modules']:
+        titre=sstr(ue["titre"]),
-        doc._push()
+        numero=sstr(ue["numero"]),
-        doc.module( coefficient="1.0", code=sstr(m['code']), 
+        type=sstr(ue["type"]),
-                    heures_td=sstr(m['heures_td']), 
-                    titre=sstr(m['titre']), abbrev=sstr(m['titre']),
-                    semestre_id=sstr(m['semestre_id']),
-                    numero=sstr(m['numero']) 
     )
-        doc._pop() # /module
+    doc.append(ue)
-    doc._pop() # /matiere
+    x_mat = ElementTree.Element(
-    doc._pop() # /ue
+        "matiere", titre=sstr(ue["titre"])
-    
+    )  # useless but necessary
-doc._pop() # /formation
+    x_ue.append(x_mat)
+    for m in ue["modules"]:
+        x_mod = ElementTree.Element(
+            "module",
+            coefficient="1.0",
+            code=sstr(m["code"]),
+            heures_td=sstr(m["heures_td"]),
+            titre=sstr(m["titre"]),
+            abbrev=sstr(m["titre"]),
+            semestre_id=sstr(m["semestre_id"]),
+            numero=sstr(m["numero"]),
+        )
+        x_mat.append(x_mod)
 
 # ---
-print 'Writing XML file: ', OUTPUT_FILENAME
+print "Writing XML file: ", OUTPUT_FILENAME
-f = open(OUTPUT_FILENAME, 'w')
+f = open(OUTPUT_FILENAME, "w")
+f.write("""<?xml version="1.0" encoding="utf-8"?>\n""")
 f.write(str(doc))
 f.close()

requirements-2.7.txt

@@ -27,7 +27,6 @@ icalendar==4.0.7
 idna==2.10
 isort==4.3.21
 itsdangerous==1.1.0
-jaxml==3.2
 Jinja2==2.11.2
 lazy-object-proxy==1.6.0
 Mako==1.1.4

tests/test_export_xml.py

@@ -0,0 +1,141 @@
+# -*- coding: UTF-8 -*
+
+"""Unit tests for XML exports
+
+Usage: python -m unittest tests.test_export_xml
+"""
+
+# ScoDoc7 utilisait jaxml, obsolete et non portée en python3
+# On teste ici les fionctions de remplacement, fournies par
+# notre nouveau module sco_xml.py
+
+from __future__ import print_function
+import os
+import re
+import sys
+import unittest
+
+sys.path.append("/mac/ScoDoc")
+
+from app.scodoc import sco_xml
+from app.scodoc.gen_tables import GenTable
+
+# Legacy function
+# import jaxml
+# from app.scodoc import sco_utils as scu
+
+# r = scu.simple_dictlist2xml([{"id": 1, "ues": [{"note": 10}, {}]}], tagname="infos")
+
+
+def xml_normalize(x):
+    "supprime espaces inutiles"
+    x = re.sub(r"\s+", " ", str(x)).strip().replace("> <", "><")
+
+
+def xmls_compare(x, y):
+    return xml_normalize(x) == xml_normalize(y)
+
+
+# expected_result est le résultat de l'ancienne fonction ScoDoc7:
+for (data, expected_result) in (
+    (
+        [{"id": 1, "ues": [{"note": 10}, {}, {"valeur": 25}]}, {"bis": 2}],
+        """<?xml version="1.0" encoding="utf-8"?>
+<infos id="1">
+    <ues note="10" />
+    <ues />
+    <ues valeur="25" />
+</infos>
+<infos bis="2" />
+""",
+    ),
+    ([], """"""),
+    (
+        ["allo"],
+        """<?xml version="1.0" encoding="utf-8"?>
+<infos code="allo" />
+""",
+    ),
+    (
+        [{}],
+        """<?xml version="1.0" encoding="utf-8"?>
+<infos />
+""",
+    ),
+    (
+        [{"x": 1}],
+        """<?xml version="1.0" encoding="utf-8"?>
+<infos x="1" />
+""",
+    ),
+    (
+        [{"y": [1, 2, 3], "x": 1}],
+        """<?xml version="1.0" encoding="utf-8"?>
+<infos x="1">
+    <y code="1" />
+    <y code="2" />
+    <y code="3" />
+</infos>
+""",
+    ),
+    (
+        [{"y": [{"x": 1}, {"y": [1, 2, 3]}], "x": 1}],
+        """<?xml version="1.0" encoding="utf-8"?>
+<infos x="1">
+    <y x="1" />
+    <y>
+        <y code="1" />
+        <y code="2" />
+        <y code="3" />
+    </y>
+</infos>
+""",
+    ),
+):
+    # x = scu.simple_dictlist2xml(data, tagname="infos")
+    y = sco_xml.simple_dictlist2xml(data, tagname="infos")
+    assert xmls_compare(expected_result, y)
+    # print("""({}, '''{}'''),""".format(data, str(x)))
+
+# test du sendXML compatible ScoDoc7
+etuds = [{"x": 1, "etuds": ["allo", "mama"]}, {"x": 2, "etuds": ["un", "deux"]}]
+# Le résultat de l'ancien print(sendXML(None, etuds, tagname="etudiants"))
+expected_result = """
+<?xml version="1.0" encoding="utf-8"?>
+<etudiants_list>
+    <etudiants x="1">
+        <etuds code="allo" />
+        <etuds code="mama" />
+    </etudiants>
+    <etudiants x="2">
+        <etuds code="un" />
+        <etuds code="deux" />
+    </etudiants>
+</etudiants_list>
+"""
+
+assert xmls_compare(
+    expected_result,
+    sco_xml.simple_dictlist2xml([{"etudiant": etuds}], tagname="etudiant_list"),
+)
+
+# ---- Tables
+T = GenTable(
+    rows=[{"nom": "Toto", "age": 26}, {"nom": "Titi", "age": 21}],
+    columns_ids=("nom", "age"),
+)
+print(T.xml())
+
+expected_result = """
+<?xml version="1.0" encoding="utf-8"?>
+<table origin="" caption="" id="gt_806883">
+    <row>
+        <nom value="Toto" />
+        <age value="26" />
+    </row>
+    <row>
+        <nom value="Titi" />
+        <age value="21" />
+    </row>
+</table>
+"""