From 5906ba62831f394ea6eca8a5ae23c09e6dc18787 Mon Sep 17 00:00:00 2001 From: Emmanuel Viennet Date: Sun, 11 Jul 2021 13:03:13 +0200 Subject: [PATCH] completed elimination of jaxml --- app/scodoc/sco_utils.py | 21 +--- app/views/absences.py | 37 +++--- app/views/notes.py | 13 +- app/views/users.py | 16 +-- config/install_debian10.sh | 3 +- misc/iscid_create_formation_from_xls.py | 159 +++++++++++++----------- requirements-2.7.txt | 1 - tests/test_export_xml.py | 141 +++++++++++++++++++++ 8 files changed, 267 insertions(+), 124 deletions(-) create mode 100644 tests/test_export_xml.py diff --git a/app/scodoc/sco_utils.py b/app/scodoc/sco_utils.py index aec0db8f..3f83e018 100644 --- a/app/scodoc/sco_utils.py +++ b/app/scodoc/sco_utils.py @@ -37,24 +37,16 @@ from hashlib import md5 import numbers import os import re -import sys +import six import six.moves._thread +import sys import time import types import six.moves.urllib.request, six.moves.urllib.parse, six.moves.urllib.error import six.moves.urllib.request, six.moves.urllib.error, six.moves.urllib.parse +from xml.etree.ElementTree import Element - -# XML generation package (apt-get install jaxml) -import jaxml # XXX - -try: - import six - - STRING_TYPES = six.string_types -except ImportError: - # fallback for very old ScoDoc instances - STRING_TYPES = bytes +STRING_TYPES = six.string_types from PIL import Image as PILImage @@ -876,9 +868,8 @@ def _sco_error_response(context, msg, format="html", REQUEST=None): raise sco_exceptions.ScoValueError(msg) elif format == "xml": REQUEST.RESPONSE.setHeader("content-type", XML_MIMETYPE) - doc = jaxml.XML_document(encoding=SCO_ENCODING) - doc.error(msg=msg) - return repr(doc) + doc = ElementTree.Element("error", msg=msg) + return sco_xml.XML_HEADER + ElementTree.tostring(doc) elif format == "json": REQUEST.RESPONSE.setHeader("content-type", JSON_MIMETYPE) return "undefined" # XXX voir quoi faire en cas d'erreur json diff --git a/app/views/absences.py b/app/views/absences.py index 295e2b0b..343a7b1b 100644 --- a/app/views/absences.py +++ b/app/views/absences.py @@ -46,16 +46,16 @@ L'API de plus bas niveau est en gros: """ -import string -import re -import time +import calendar +import cgi import datetime import dateutil import dateutil.parser -import calendar +import re import six.moves.urllib.request, six.moves.urllib.parse, six.moves.urllib.error -import cgi -import jaxml +import string +import time +from xml.etree import ElementTree from flask import g from flask import current_app @@ -93,6 +93,7 @@ from app.scodoc import sco_groups from app.scodoc import sco_groups_view from app.scodoc import sco_moduleimpl from app.scodoc import sco_preferences +from app.scodoc import sco_xml CSSSTYLES = html_sco_header.BOOTSTRAP_MULTISELECT_CSS @@ -1505,22 +1506,22 @@ def XMLgetAbsEtud(context, beg_date="", end_date="", REQUEST=None): Abs = sco_abs.ListeAbsDate(context, etud["etudid"], beg_date, end_date) REQUEST.RESPONSE.setHeader("content-type", scu.XML_MIMETYPE) - doc = jaxml.XML_document(encoding=scu.SCO_ENCODING) - doc.absences(etudid=etud["etudid"], beg_date=beg_date, end_date=end_date) - doc._push() + doc = ElementTree.Element( + "absences", etudid=etud["etudid"], beg_date=beg_date, end_date=end_date + ) for a in Abs: if a["estabs"]: # ne donne pas les justifications si pas d'absence - doc._push() - doc.abs( - begin=a["begin"], - end=a["end"], - description=a["description"], - justified=a["estjust"], + doc.append( + ElementTree.Element( + "abs", + begin=a["begin"], + end=a["end"], + description=a["description"], + justified=a["estjust"], + ) ) - doc._pop() - doc._pop() log("XMLgetAbsEtud (%gs)" % (time.time() - t0)) - return repr(doc) + return sco_xml.XML_HEADER + ElementTree.tostring(doc) context.populate(globals()) diff --git a/app/views/notes.py b/app/views/notes.py index d95d8020..62a33e57 100644 --- a/app/views/notes.py +++ b/app/views/notes.py @@ -33,9 +33,9 @@ Emmanuel Viennet, 2021 import sys import time import datetime -import jaxml import pprint from operator import itemgetter +from xml.etree import ElementTree from flask import url_for, g from flask import current_app @@ -128,6 +128,7 @@ from app.scodoc import sco_tag_module from app.scodoc import sco_ue_external from app.scodoc import sco_undo_notes from app.scodoc import sco_users +from app.scodoc import sco_xml from app.scodoc.gen_tables import GenTable from app.scodoc.sco_pdf import PDFLOCK from app.scodoc.sco_permissions import Permission @@ -650,13 +651,11 @@ def XMLgetFormsemestres(context, etape_apo=None, formsemestre_id=None, REQUEST=N args["formsemestre_id"] = formsemestre_id if REQUEST: REQUEST.RESPONSE.setHeader("content-type", scu.XML_MIMETYPE) - doc = jaxml.XML_document(encoding=scu.SCO_ENCODING) - doc.formsemestrelist() + doc = ElementTree.Element("formsemestrelist") for sem in sco_formsemestre.do_formsemestre_list(context, args=args): - doc._push() - doc.formsemestre(sem) - doc._pop() - return repr(doc) + doc.append("formsemestre", **sem) + + return sco_xml.XML_HEADER + ElementTree.tostring(doc) sco_publish( diff --git a/app/views/users.py b/app/views/users.py index 4d72c293..c57cbddc 100644 --- a/app/views/users.py +++ b/app/views/users.py @@ -34,7 +34,7 @@ Vues s'appuyant sur auth et sco_users Emmanuel Viennet, 2021 """ import re -import jaxml +from xml.etree import ElementTree from flask import g from flask_login import current_user @@ -54,6 +54,7 @@ from app.decorators import ( from app.scodoc import html_sco_header from app.scodoc import sco_users from app.scodoc import sco_utils as scu +from app.scodoc import sco_xml from app.scodoc.notes_log import log from app.scodoc.sco_exceptions import AccessDenied, ScoValueError from app.scodoc.sco_permissions_check import can_handle_passwd @@ -341,7 +342,7 @@ def create_user_form(context, REQUEST, user_name=None, edit=0): edit = 0 try: force = int(vals["force"][0]) - except: + except (ValueError, TypeError): force = 0 if edit: @@ -471,13 +472,12 @@ def get_user_list_xml(dept=None, start="", limit=25, REQUEST=None): ] if REQUEST: REQUEST.RESPONSE.setHeader("content-type", scu.XML_MIMETYPE) - doc = jaxml.XML_document(encoding=scu.SCO_ENCODING) - doc.results() + doc = ElementTree.Element("results") for user in userlist[:limit]: - doc._push() - doc.rs(user.get_nomplogin(), id=user.id, info="") - doc._pop() - return repr(doc) + x_rs = ElementTree.Element("rs", id=user.id, info="") + x_rs.text = user.get_nomplogin() + doc.append(x_rs) + return sco_xml.XML_HEADER + ElementTree.tostring(doc) @bp.route("/form_change_password") diff --git a/config/install_debian10.sh b/config/install_debian10.sh index 9269af54..fd80ef41 100755 --- a/config/install_debian10.sh +++ b/config/install_debian10.sh @@ -87,9 +87,8 @@ apt-get -y install postgresql apt-get -y install graphviz # ------------ INSTALL DES EXTENSIONS PYTHON (2.7) -# XXX to fix: pip in our env +# ScoDoc8 uses pip in our env apt-get -y install python-docutils -apt-get -y install python-jaxml apt-get -y install python-psycopg2 apt-get -y install python-pyrss2gen apt-get -y install python-pil python-reportlab diff --git a/misc/iscid_create_formation_from_xls.py b/misc/iscid_create_formation_from_xls.py index b41a9d9a..4ab38a74 100644 --- a/misc/iscid_create_formation_from_xls.py +++ b/misc/iscid_create_formation_from_xls.py @@ -6,37 +6,38 @@ # XXX TODO : a tester et moderniser (ects, verifier champs, python 3, importer codes depuis ScoDoc ?) import os, sys, pdb, pprint -from openpyxl import load_workbook # apt-get install python-openpyxl -import jaxml -SCO_ENCODING = 'utf-8' +from openpyxl import load_workbook # apt-get install python-openpyxl +from xml.etree import ElementTree + +SCO_ENCODING = "utf-8" INPUT_FILENAME = "/tmp/Bachelor.xlsx" -OUTPUT_FILENAME= os.path.splitext(INPUT_FILENAME)[0] + '.xml' +OUTPUT_FILENAME = os.path.splitext(INPUT_FILENAME)[0] + ".xml" -FIRST_SHEET_IDX=1 # saute première feuille du classeur +FIRST_SHEET_IDX = 1 # saute première feuille du classeur # Code de ScoDoc (sco_utils.py) -UE_STANDARD = 0 # UE "fondamentale" -UE_SPORT = 1 # bonus "sport" -UE_STAGE_LP = 2 # ue "projet tuteuré et stage" dans les Lic. Pro. -UE_ELECTIVE = 4 # UE "élective" dans certains parcours (UCAC?, ISCID) -UE_PROFESSIONNELLE = 5 # UE "professionnelle" (ISCID, ...) +UE_STANDARD = 0 # UE "fondamentale" +UE_SPORT = 1 # bonus "sport" +UE_STAGE_LP = 2 # ue "projet tuteuré et stage" dans les Lic. Pro. +UE_ELECTIVE = 4 # UE "élective" dans certains parcours (UCAC?, ISCID) +UE_PROFESSIONNELLE = 5 # UE "professionnelle" (ISCID, ...) # Code du fichier Excel: -UE_TYPE2CODE = { u'UE F' : UE_STANDARD, u'UE E' : UE_ELECTIVE } +UE_TYPE2CODE = {u"UE F": UE_STANDARD, u"UE E": UE_ELECTIVE} # Lecture du fichier Excel UE = [] wb = load_workbook(filename=INPUT_FILENAME) -#print wb.get_sheet_names() +# print wb.get_sheet_names() for sheet_name in wb.get_sheet_names()[FIRST_SHEET_IDX:]: - print 'Importing sheet %s' % sheet_name + print "Importing sheet %s" % sheet_name sheet = wb.get_sheet_by_name(sheet_name) # Avance jusqu'à trouver le titre 'CODE' en premiere colonne - i=0 - while i < len(sheet.rows) and sheet.rows[i][0].value != 'CODE': + i = 0 + while i < len(sheet.rows) and sheet.rows[i][0].value != "CODE": i = i + 1 i = i + 1 @@ -48,81 +49,93 @@ for sheet_name in wb.get_sheet_names()[FIRST_SHEET_IDX:]: if ue: UE.append(ue) # creation UE - acronyme = code # ici l'acronyme d'UE est le code du module - if not acronyme and (i < len(sheet.rows)-1): - acronyme = sheet.rows[i+1][0].value # code module sur ligne suivante - #print acronyme - if acronyme: # tres specifique: deduit l'acronyme d'UE du code module - parts = acronyme.split(u'-') - parts[-1] = parts[-1][-1] # ne garde que le dernier chiffre - acronyme = u'-'.join(parts) # B1-LV1-EN1 -> B1-LV1-1 - #print '->', acronyme + acronyme = code # ici l'acronyme d'UE est le code du module + if not acronyme and (i < len(sheet.rows) - 1): + acronyme = sheet.rows[i + 1][0].value # code module sur ligne suivante + # print acronyme + if acronyme: # tres specifique: deduit l'acronyme d'UE du code module + parts = acronyme.split(u"-") + parts[-1] = parts[-1][-1] # ne garde que le dernier chiffre + acronyme = u"-".join(parts) # B1-LV1-EN1 -> B1-LV1-1 + # print '->', acronyme if not acronyme: - acronyme = sheet.rows[i][3].value # fallback: titre - ue = { 'acronyme' : acronyme, - 'titre' : sheet.rows[i][3].value, - 'ects' : sheet.rows[i][5].value or u"", - 'type' : UE_TYPE2CODE[type_ue], - 'numero' : (sheet.rows[i][1].value or 0)*1000 + i*10, - 'modules' : [] - } + acronyme = sheet.rows[i][3].value # fallback: titre + ue = { + "acronyme": acronyme, + "titre": sheet.rows[i][3].value, + "ects": sheet.rows[i][5].value or u"", + "type": UE_TYPE2CODE[type_ue], + "numero": (sheet.rows[i][1].value or 0) * 1000 + i * 10, + "modules": [], + } i_ue = i if code: - ue['modules'].append( { - 'code' : code, - 'heures_td' : sheet.rows[i_ue][4].value or u"", - 'titre' : sheet.rows[i][3].value, - 'semestre_id' : sheet.rows[i][1].value, - 'numero' : i*10 - } ) + ue["modules"].append( + { + "code": code, + "heures_td": sheet.rows[i_ue][4].value or u"", + "titre": sheet.rows[i][3].value, + "semestre_id": sheet.rows[i][1].value, + "numero": i * 10, + } + ) - i += 1 # next line + i += 1 # next line if ue: UE.append(ue) def sstr(s): - if type(s) is type(u''): + if type(s) is type(u""): return s.encode(SCO_ENCODING) else: return str(s) -# ----- Write to XML -doc = jaxml.XML_document( encoding=SCO_ENCODING ) -doc._push() -doc.formation( acronyme="Bachelor ISCID", - code_specialite="", - type_parcours="1001", - titre_officiel="Bachelor ISCID", - formation_code="FCOD4", - version="1", - titre="Bachelor ISCID", - formation_id="FORM115" - ) +# ----- Write to XML +doc = ElementTree.Element( + "formation", + acronyme="Bachelor ISCID", + code_specialite="", + type_parcours="1001", + titre_officiel="Bachelor ISCID", + formation_code="FCOD4", + version="1", + titre="Bachelor ISCID", + formation_id="FORM115", +) for ue in UE: - doc._push() - doc.ue( acronyme=sstr(ue['acronyme']), ects=sstr(ue['ects']), titre=sstr(ue['titre']), numero=sstr(ue['numero']), type=sstr(ue['type']) ) - doc._push() - doc.matiere( titre=sstr(ue['titre']) ) # useless but necessary - for m in ue['modules']: - doc._push() - doc.module( coefficient="1.0", code=sstr(m['code']), - heures_td=sstr(m['heures_td']), - titre=sstr(m['titre']), abbrev=sstr(m['titre']), - semestre_id=sstr(m['semestre_id']), - numero=sstr(m['numero']) - ) - doc._pop() # /module - doc._pop() # /matiere - doc._pop() # /ue - -doc._pop() # /formation + x_ue = ElementTree.Element( + "ue", + acronyme=sstr(ue["acronyme"]), + ects=sstr(ue["ects"]), + titre=sstr(ue["titre"]), + numero=sstr(ue["numero"]), + type=sstr(ue["type"]), + ) + doc.append(ue) + x_mat = ElementTree.Element( + "matiere", titre=sstr(ue["titre"]) + ) # useless but necessary + x_ue.append(x_mat) + for m in ue["modules"]: + x_mod = ElementTree.Element( + "module", + coefficient="1.0", + code=sstr(m["code"]), + heures_td=sstr(m["heures_td"]), + titre=sstr(m["titre"]), + abbrev=sstr(m["titre"]), + semestre_id=sstr(m["semestre_id"]), + numero=sstr(m["numero"]), + ) + x_mat.append(x_mod) -#--- -print 'Writing XML file: ', OUTPUT_FILENAME -f = open(OUTPUT_FILENAME, 'w') +# --- +print "Writing XML file: ", OUTPUT_FILENAME +f = open(OUTPUT_FILENAME, "w") +f.write("""\n""") f.write(str(doc)) f.close() diff --git a/requirements-2.7.txt b/requirements-2.7.txt index 5f6d9e96..4ce146c0 100644 --- a/requirements-2.7.txt +++ b/requirements-2.7.txt @@ -27,7 +27,6 @@ icalendar==4.0.7 idna==2.10 isort==4.3.21 itsdangerous==1.1.0 -jaxml==3.2 Jinja2==2.11.2 lazy-object-proxy==1.6.0 Mako==1.1.4 diff --git a/tests/test_export_xml.py b/tests/test_export_xml.py new file mode 100644 index 00000000..11c438ce --- /dev/null +++ b/tests/test_export_xml.py @@ -0,0 +1,141 @@ +# -*- coding: UTF-8 -* + +"""Unit tests for XML exports + +Usage: python -m unittest tests.test_export_xml +""" + +# ScoDoc7 utilisait jaxml, obsolete et non portée en python3 +# On teste ici les fionctions de remplacement, fournies par +# notre nouveau module sco_xml.py + +from __future__ import print_function +import os +import re +import sys +import unittest + +sys.path.append("/mac/ScoDoc") + +from app.scodoc import sco_xml +from app.scodoc.gen_tables import GenTable + +# Legacy function +# import jaxml +# from app.scodoc import sco_utils as scu + +# r = scu.simple_dictlist2xml([{"id": 1, "ues": [{"note": 10}, {}]}], tagname="infos") + + +def xml_normalize(x): + "supprime espaces inutiles" + x = re.sub(r"\s+", " ", str(x)).strip().replace("> <", "><") + + +def xmls_compare(x, y): + return xml_normalize(x) == xml_normalize(y) + + +# expected_result est le résultat de l'ancienne fonction ScoDoc7: +for (data, expected_result) in ( + ( + [{"id": 1, "ues": [{"note": 10}, {}, {"valeur": 25}]}, {"bis": 2}], + """ + + + + + + +""", + ), + ([], """"""), + ( + ["allo"], + """ + +""", + ), + ( + [{}], + """ + +""", + ), + ( + [{"x": 1}], + """ + +""", + ), + ( + [{"y": [1, 2, 3], "x": 1}], + """ + + + + + +""", + ), + ( + [{"y": [{"x": 1}, {"y": [1, 2, 3]}], "x": 1}], + """ + + + + + + + + +""", + ), +): + # x = scu.simple_dictlist2xml(data, tagname="infos") + y = sco_xml.simple_dictlist2xml(data, tagname="infos") + assert xmls_compare(expected_result, y) + # print("""({}, '''{}'''),""".format(data, str(x))) + +# test du sendXML compatible ScoDoc7 +etuds = [{"x": 1, "etuds": ["allo", "mama"]}, {"x": 2, "etuds": ["un", "deux"]}] +# Le résultat de l'ancien print(sendXML(None, etuds, tagname="etudiants")) +expected_result = """ + + + + + + + + + + + +""" + +assert xmls_compare( + expected_result, + sco_xml.simple_dictlist2xml([{"etudiant": etuds}], tagname="etudiant_list"), +) + +# ---- Tables +T = GenTable( + rows=[{"nom": "Toto", "age": 26}, {"nom": "Titi", "age": 21}], + columns_ids=("nom", "age"), +) +print(T.xml()) + +expected_result = """ + + + + + + + + + + +
+"""