Fix unicode strings for Python3

This commit is contained in:
Emmanuel Viennet 2021-07-12 15:13:10 +02:00
parent 99279a96bd
commit 0ec41c48f3
22 changed files with 88 additions and 134 deletions

View File

@ -41,32 +41,28 @@ class ZRequest(object):
"Emulating Zope 2 REQUEST"
def __init__(self):
self.URL = (
request.base_url
) # .encode("utf-8") # necessaire pour ScoDoc 8 en Python 2 #sco8
self.URL = request.base_url
self.URL0 = self.URL
self.BASE0 = request.url_root # .encode("utf-8")
self.QUERY_STRING = request.query_string # .encode("utf-8")
self.REQUEST_METHOD = request.method # .encode("utf-8")
self.BASE0 = request.url_root
self.QUERY_STRING = request.query_string.decode(
"utf-8"
) # query_string is bytes
self.REQUEST_METHOD = request.method
self.AUTHENTICATED_USER = current_user
self.REMOTE_ADDR = request.remote_addr
if request.method == "POST":
self.form = request.form # xxx encode en utf-8 !
# Encode en utf-8 pour ScoDoc8 #sco8
# self.form = {k: v.encode("utf-8") for (k, v) in request.form.items()}
# request.form is a werkzeug.datastructures.ImmutableMultiDict
# must copy to get a mutable version (needed by TrivialFormulator)
self.form = request.form.copy()
if request.files:
# Add files in form: must copy to get a mutable version
# request.form is a werkzeug.datastructures.ImmutableMultiDict
# self.form = self.form.copy()
# Add files in form:
self.form.update(request.files)
# self.cf = request.form.copy()
for k in request.form:
if k.endswith(":list"):
self.form[k[:-5]] = request.form.getlist(k)
elif request.method == "GET":
self.form = {k: v for (k, v) in request.args.items()} # forme python3
# Encode en utf-8 pour ScoDoc8 #sco8
# self.form = {k: v.encode("utf-8") for (k, v) in request.args.items()}
self.form = {k: v for (k, v) in request.args.items()}
self.RESPONSE = ZResponse()
def __str__(self):
@ -89,7 +85,7 @@ class ZResponse(object):
def redirect(self, url):
current_app.logger.debug("ZResponse redirect to:" + str(url))
return flask.redirect(url) # .decode("utf-8")) # http 302 # #sco8 unicode
return flask.redirect(url) # http 302
def setHeader(self, header, value):
self.headers[header.lower()] = value
@ -100,7 +96,7 @@ def permission_required(permission):
@wraps(f)
def decorated_function(*args, **kwargs):
if "scodoc_dept" in kwargs:
g.scodoc_dept = kwargs["scodoc_dept"] # .encode("utf-8") # sco8
g.scodoc_dept = kwargs["scodoc_dept"]
del kwargs["scodoc_dept"]
# current_app.logger.info(
# "permission_required: %s in %s" % (permission, g.scodoc_dept)
@ -148,7 +144,7 @@ def scodoc7func(context):
return func(*args, **kwargs)
#
if "scodoc_dept" in kwargs:
g.scodoc_dept = kwargs["scodoc_dept"] # .encode("utf-8") # sco8
g.scodoc_dept = kwargs["scodoc_dept"]
del kwargs["scodoc_dept"]
elif not hasattr(g, "scodoc_dept"):
g.scodoc_dept = None
@ -174,12 +170,9 @@ def scodoc7func(context):
elif arg_name == "context":
pos_arg_values.append(context)
else:
# XXX Convert to regular string for ScoDoc8/Python 2 #py3
# if isinstance(req_args[arg_name], str):
# pos_arg_values.append(req_args[arg_name].encode("utf-8"))
# else:
pos_arg_values.append(req_args[arg_name])
current_app.logger.info("pos_arg_values=%s" % pos_arg_values)
current_app.logger.info("req_args=%s" % req_args)
# Add keyword arguments
if nb_default_args:
for arg_name in argspec.args[-nb_default_args:]:
@ -187,10 +180,6 @@ def scodoc7func(context):
kwargs[arg_name] = REQUEST
elif arg_name in req_args:
# set argument kw optionnel
# XXX Convert to regular string for ScoDoc8/Python 2 #py3
# if isinstance(req_args[arg_name], str):
# kwargs[arg_name] = req_args[arg_name].encode("utf-8")
# else:
kwargs[arg_name] = req_args[arg_name]
current_app.logger.info(
"scodoc7func_decorator: top_level=%s, pos_arg_values=%s, kwargs=%s"

View File

@ -332,7 +332,7 @@ def sco_header(
# div pour affichage messages temporaires
H.append('<div id="sco_msg" class="head_message"></div>')
#
return scu.sco8_join(H, sep="")
return "".join(H)
def sco_footer(context, REQUEST=None):

View File

@ -103,7 +103,7 @@ def make_menu(title, items, css_class="", alone=False):
args = item.get("args", {})
item["urlq"] = url_for(
item["endpoint"], scodoc_dept=g.scodoc_dept, **args
).encode(scu.SCO_ENCODING)
)
else:
item["urlq"] = "#"
item["attr"] = item.get("attr", "")

View File

@ -9,10 +9,11 @@ import traceback
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.header import Header
from flask import current_app
# Simple & stupid file logguer, used only to debug
# (logging to SQL is done in scolog)
"""Simple & stupid file logguer, used only to debug
(logging to SQL is done in scolog)
"""
LOG_FILENAME = "notes.log" # empty to disable logging
DEFAULT_LOG_DIR = "/tmp" # clients should call set_log_directory to change this
@ -48,12 +49,10 @@ class _logguer(object):
dept = retreive_dept()
if dept:
dept = " (%s)" % dept
self.file.write(
"[%s]%s %s\n" % (time.strftime("%a %b %d %H:%M:%S %Y"), dept, msg)
)
# if not dept:
# import traceback
# traceback.print_stack(file=self.file) # hunt missing REQUESTS
msg = dept + " " + msg
self.file.write("[%s]%s\n" % (time.strftime("%a %b %d %H:%M:%S %Y"), msg))
if current_app:
current_app.logger.info(msg)
self.file.flush()

View File

@ -390,10 +390,7 @@ class NotesTable(object):
return (
scu.strupper(etud["nom_usuel"] or etud["nom"])
+ " "
+ etud["prenom"]
.decode(scu.SCO_ENCODING)
.capitalize()[:2]
.encode(scu.SCO_ENCODING)
+ etud["prenom"].capitalize()[:2]
+ "."
)

View File

@ -1,8 +1,7 @@
# -*- mode: python -*-
# -*- coding: utf-8 -*-
import string
import html
import traceback
import psycopg2
import psycopg2.pool
@ -13,10 +12,9 @@ from flask import g
import app.scodoc.sco_utils as scu
from app.scodoc.notes_log import log
from app.scodoc.sco_exceptions import ScoException, ScoValueError, NoteProcessError
from cgi import escape
import datetime
quote_html = escape
quote_html = html.escape
def quote_dict(d):

View File

@ -74,13 +74,8 @@ def get_code_latex_from_scodoc_preference(
et s'assure qu'il est renvoyé au format unicode
"""
template_latex = sco_preferences.get_preference(context, champ, formsemestre_id)
# Conversion du template en unicode:
if template_latex:
template_latex = template_latex.decode(scu.SCO_ENCODING)
else:
template_latex = u"" # EV: preference non définie (None)
return template_latex
return template_latex or ""
# ----------------------------------------------------------------------------------------
@ -123,15 +118,9 @@ def comp_latex_parcourstimeline(etudiant, promo, taille=17):
\\end{parcourstimeline}
"""
reslatex = codelatexDebut
reslatex = reslatex.replace(
"**debut**", etudiant["entree"].decode(scu.SCO_ENCODING)
)
reslatex = reslatex.replace(
"**fin**", str(etudiant["promo"]).decode(scu.SCO_ENCODING)
)
reslatex = reslatex.replace(
"**nbreSemestres**", str(etudiant["nbSemestres"]).decode(scu.SCO_ENCODING)
)
reslatex = reslatex.replace("**debut**", etudiant["entree"])
reslatex = reslatex.replace("**fin**", str(etudiant["promo"]))
reslatex = reslatex.replace("**nbreSemestres**", str(etudiant["nbSemestres"]))
# Tri du parcours par ordre croissant : de la forme descr, nom sem date-date
parcours = etudiant["parcours"][::-1] # EV: XXX je ne comprend pas ce commentaire ?
@ -218,11 +207,9 @@ def get_code_latex_avis_etudiant(
# Les tags "simples": par ex. nom, prenom, civilite, ...
else:
if tag_latex in donnees_etudiant:
valeur = donnees_etudiant[tag_latex].decode(scu.SCO_ENCODING)
valeur = donnees_etudiant[tag_latex]
elif tag_latex in prefs: # les champs **NomResponsablePE**, ...
valeur = pe_tools.escape_for_latex(prefs[tag_latex]).decode(
scu.SCO_ENCODING
)
valeur = pe_tools.escape_for_latex(prefs[tag_latex])
# Vérification des pb d'encodage (debug)
# assert isinstance(tag_latex, unicode)
@ -251,7 +238,7 @@ def get_annotation_PE(context, etudid, tag_annotation_pe):
exp = re.compile(r"^" + tag_annotation_pe)
for a in annotations:
commentaire = scu.unescape_html(a["comment"]).decode(scu.SCO_ENCODING)
commentaire = scu.unescape_html(a["comment"])
if exp.match(commentaire): # tag en début de commentaire ?
a["comment_u"] = commentaire # unicode, HTML non quoté
annotationsPE.append(
@ -383,11 +370,7 @@ def get_bilanParTag(donnees_etudiant, groupe="groupe"):
for (i, ligne_val) in enumerate(valeurs["note"]):
titre = lignes[i] # règle le pb d'encodage
code_latex += (
u"\\textbf{"
+ titre.decode(scu.SCO_ENCODING)
+ u"} & "
+ " & ".join(ligne_val)
+ u"\\\\ \n"
u"\\textbf{" + titre + u"} & " + " & ".join(ligne_val) + u"\\\\ \n"
)
code_latex += (
u" & "
@ -412,11 +395,9 @@ def get_avis_poursuite_par_etudiant(
if pe_tools.PE_DEBUG:
pe_tools.pe_print(jury.syntheseJury[etudid]["nom"] + " " + etudid)
civilite_str = jury.syntheseJury[etudid]["civilite_str"].decode(scu.SCO_ENCODING)
nom = jury.syntheseJury[etudid]["nom"].replace(" ", "-").decode(scu.SCO_ENCODING)
prenom = (
jury.syntheseJury[etudid]["prenom"].replace(" ", "-").decode(scu.SCO_ENCODING)
)
civilite_str = jury.syntheseJury[etudid]["civilite_str"]
nom = jury.syntheseJury[etudid]["nom"].replace(" ", "-")
prenom = jury.syntheseJury[etudid]["prenom"].replace(" ", "-")
nom_fichier = (
u"avis_poursuite_"
@ -538,9 +519,7 @@ def table_syntheseAnnotationPE(context, syntheseJury, tag_annotation_pe):
annotationPE = get_annotation_PE(
context, etudid, tag_annotation_pe=tag_annotation_pe
)
row["Annotation PE"] = (
annotationPE.encode(scu.SCO_ENCODING) if annotationPE else ""
)
row["Annotation PE"] = annotationPE if annotationPE else ""
rows.append(row)
T = GenTable(

View File

@ -106,7 +106,7 @@ def pe_view_sem_recap(
# template fourni via le formulaire Web
if avis_tmpl_file:
template_latex = avis_tmpl_file.read()
template_latex = template_latex.decode(scu.SCO_ENCODING)
template_latex = template_latex
else:
# template indiqué dans préférences ScoDoc ?
template_latex = pe_avislatex.get_code_latex_from_scodoc_preference(
@ -123,7 +123,7 @@ def pe_view_sem_recap(
# template fourni via le formulaire Web
if footer_tmpl_file:
footer_latex = footer_tmpl_file.read()
footer_latex = footer_latex.decode(scu.SCO_ENCODING)
footer_latex = footer_latex
else:
footer_latex = pe_avislatex.get_code_latex_from_scodoc_preference(
context, formsemestre_id, champ="pe_avis_latex_footer"

View File

@ -93,15 +93,15 @@ class ddmmyyyy(object):
return
try:
if fmt == "ddmmyyyy":
self.day, self.month, self.year = string.split(date, "/")
self.day, self.month, self.year = date.split("/")
elif fmt == "iso":
self.year, self.month, self.day = string.split(date, "-")
self.year, self.month, self.day = date.split("-")
else:
raise ValueError("invalid format spec. (%s)" % fmt)
self.year = string.atoi(self.year)
self.month = string.atoi(self.month)
self.day = string.atoi(self.day)
except:
self.year = int(self.year)
self.month = int(self.month)
self.day = int(self.day)
except ValueError:
raise ScoValueError("date invalide: %s" % date)
# accept years YYYY or YY, uses 1970 as pivot
if self.year < 1970:
@ -297,7 +297,7 @@ def YearTable(
month = 1
year = year + 1
T.append("</table>")
return string.join(T, "\n")
return "\n".join(T)
def list_abs_in_range(
@ -876,7 +876,7 @@ def MonthTableBody(
if href or descr:
cc.append("</a>")
cc.append("</td>")
cell = string.join(cc, "")
cell = "".join(cc)
if day == "D":
monday = monday.next_day(7)
if (
@ -968,8 +968,8 @@ def MonthTableBody(
if href or descr:
cc.append("</a>")
cc.append("</td>\n")
T.append(string.join(cc, "") + "</tr>")
return string.join(T, "\n")
T.append("".join(cc) + "</tr>")
return "\n".join(T)
# --------------------------------------------------------------------

View File

@ -194,14 +194,16 @@ def fix_data_encoding(
default_source_encoding,
dest_encoding,
)
text = text.decode(default_source_encoding).encode(dest_encoding)
text = text.decode(default_source_encoding).encode(
dest_encoding
) # XXX #py3 #sco8 à tester
else:
if detected_encoding != dest_encoding:
message = "converting from detected %s to %s" % (
detected_encoding,
dest_encoding,
)
text = text.decode(detected_encoding).encode(dest_encoding)
text = text.decode(detected_encoding).encode(dest_encoding) # XXX
return text, message
@ -782,7 +784,9 @@ class ApoData(object):
if not data:
raise FormatError("Fichier Apogée vide !")
data_utf8 = data.decode(APO_INPUT_ENCODING).encode(scu.SCO_ENCODING)
data_utf8 = data.decode(APO_INPUT_ENCODING).encode(
scu.SCO_ENCODING
) # XXX #sco8
f = StringIOFileLineWrapper(data_utf8) # pour traiter comme un fichier
# check that we are at the begining of Apogee CSV
line = f.readline().strip()
@ -1316,7 +1320,7 @@ def export_csv_to_apogee(
)
log(logf.getvalue()) # sortie aussi sur le log ScoDoc
csv_data = f.getvalue().decode(scu.SCO_ENCODING).encode(APO_OUTPUT_ENCODING)
csv_data = f.getvalue().decode(scu.SCO_ENCODING).encode(APO_OUTPUT_ENCODING) # XXX
# Write data to ZIP
dest_zip.writestr(csv_filename, csv_data)

View File

@ -124,7 +124,7 @@ class BaseArchiver(object):
base
+ "[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]-[0-9][0-9]-[0-9][0-9]-[0-9][0-9]"
)
dirs = [os.path.join(base, d).encode(scu.SCO_ENCODING) for d in dirs] # sco8
dirs = [os.path.join(base, d) for d in dirs]
dirs = [d for d in dirs if os.path.isdir(d) and os.listdir(d)] # non empty dirs
dirs.sort()
return dirs
@ -150,7 +150,9 @@ class BaseArchiver(object):
finally:
scu.GSL.release()
files.sort()
return [f.encode(scu.SCO_ENCODING) for f in files if f and f[0] != "_"] # sco8
return [
f.encode(scu.SCO_ENCODING) for f in files if f and f[0] != "_"
] # sco8 XXX
def get_archive_name(self, archive_id):
"""name identifying archive, to be used in web URLs"""

View File

@ -84,7 +84,7 @@ class CacheNotesTable(object):
raise NoteProcessError("problem with notes cache")
def get_NotesTable(self, context, formsemestre_id): # >
import notes_table
from app.scodoc import notes_table
try:
self.acquire()

View File

@ -354,7 +354,7 @@ def itemsuivi_tag_list(context, itemsuivi_id):
def itemsuivi_tag_search(context, term, REQUEST=None):
"""List all used tag names (for auto-completion)"""
# restrict charset to avoid injections
if not scu.ALPHANUM_EXP.match(term.decode(scu.SCO_ENCODING)):
if not scu.ALPHANUM_EXP.match(term):
data = []
else:
r = ndb.SimpleDictFetch(

View File

@ -142,12 +142,10 @@ def group_edt_json(context, group_id, start="", end="", REQUEST=None):
events = [e for e in cal.walk() if e.name == "VEVENT"]
J = []
for e in events:
# if e['X-GROUP-ID'].encode('utf-8').strip() == edt_group_name:
# if e['X-GROUP-ID'].strip() == edt_group_name:
if "DESCRIPTION" in e:
d = {
"title": e.decoded(
"DESCRIPTION"
), # + '/' + e['X-GROUP-ID'].encode('utf-8'),
"title": e.decoded("DESCRIPTION"), # + '/' + e['X-GROUP-ID'],
"start": e.decoded("dtstart").isoformat(),
"end": e.decoded("dtend").isoformat(),
}
@ -159,7 +157,7 @@ def group_edt_json(context, group_id, start="", end="", REQUEST=None):
"""XXX
for e in events:
if 'DESCRIPTION' in e:
print e.decoded('DESCRIPTION').encode('utf-8')
print e.decoded('DESCRIPTION')
"""

View File

@ -33,15 +33,14 @@ from operator import itemgetter
import app.scodoc.sco_utils as scu
import app.scodoc.notesdb as ndb
from notesdb import ScoDocCursor, EditableTable, DateISOtoDMY, DateDMYtoISO
from app.scodoc.notesdb import ScoDocCursor, EditableTable, DateISOtoDMY, DateDMYtoISO
def _format_nom(nom):
"formatte nom (filtre en entree db) d'une entreprise"
if not nom:
return nom
nom = nom.decode(scu.SCO_ENCODING)
return (nom[0].upper() + nom[1:]).encode(scu.SCO_ENCODING)
return nom[0].upper() + nom[1:]
class EntreprisesEditor(EditableTable):

View File

@ -667,7 +667,7 @@ def view_apo_csv_download_and_store(context, etape_apo="", semset_id="", REQUEST
)
# here, data is utf8
# but we store and generate latin1 files, to ease further import in Apogée
data = data.decode(APO_PORTAL_ENCODING).encode(APO_INPUT_ENCODING)
data = data.decode(APO_PORTAL_ENCODING).encode(APO_INPUT_ENCODING) # XXX #py3
return view_apo_csv_store(
context, semset_id, data=data, autodetect=False, REQUEST=REQUEST
)

View File

@ -299,9 +299,9 @@ def check_nom_prenom(cnx, nom="", prenom="", etudid=None):
"""
if not nom or (not prenom and not scu.CONFIG.ALLOW_NULL_PRENOM):
return False, 0
nom = nom.decode(SCO_ENCODING).lower().strip().encode(SCO_ENCODING)
nom = nom.lower().strip()
if prenom:
prenom = prenom.decode(SCO_ENCODING).lower().strip().encode(SCO_ENCODING)
prenom = prenom.lower().strip()
# Don't allow some special cars (eg used in sql regexps)
if scu.FORBIDDEN_CHARS_EXP.search(nom) or scu.FORBIDDEN_CHARS_EXP.search(prenom):
return False, 0

View File

@ -217,9 +217,7 @@ def search_etud_by_name(context, term, REQUEST=None):
# term = scu.strupper(term) # conserve les accents
term = term.upper()
if (
not scu.ALPHANUM_EXP.match(
term.decode(scu.SCO_ENCODING)
) # n'autorise pas les caractères spéciaux
not scu.ALPHANUM_EXP.match(term) # n'autorise pas les caractères spéciaux
and not may_be_nip
):
data = []

View File

@ -144,6 +144,7 @@ def _menuScolarite(context, authuser, sem, etudid):
def ficheEtud(context, etudid=None, REQUEST=None):
"fiche d'informations sur un etudiant"
log("XXX ficheEtud etudid=%s" % etudid)
authuser = REQUEST.AUTHENTICATED_USER
cnx = ndb.GetDBConnexion()
if etudid and REQUEST:

View File

@ -203,7 +203,7 @@ class ModuleTag(ScoTag):
def module_tag_search(context, term, REQUEST=None):
"""List all used tag names (for auto-completion)"""
# restrict charset to avoid injections
if not scu.ALPHANUM_EXP.match(term.decode(scu.SCO_ENCODING)):
if not scu.ALPHANUM_EXP.match(term):
data = []
else:
r = ndb.SimpleDictFetch(

View File

@ -375,7 +375,7 @@ def user_info_page(context, user_name=None, REQUEST=None):
'<p><a class="stdlink" href="%s">Liste de tous les utilisateurs</a></p>'
% url_for("users.index_html", scodoc_dept=g.scodoc_dept)
)
return scu.sco8_join(H) + F
return "\n".join(H) + F
def check_modif_user(edit, user_name="", nom="", prenom="", email="", roles=[]):

View File

@ -65,10 +65,6 @@ from app.scodoc import sco_exceptions
from app.scodoc import sco_xml
from app.scodoc import VERSION
# ----- TEMPORAIRE POUR MIGRATION SCODOC7 -> SCODOC8 avant python3
def sco8_join(L, sep="\n"): # sco8
return sep.join([x if not isinstance(x, str) else x.encode("utf-8") for x in L])
# ----- CALCUL ET PRESENTATION DES NOTES
NOTES_PRECISION = 1e-4 # evite eventuelles erreurs d'arrondis
@ -344,7 +340,7 @@ def ScoURL():
"""
return url_for("scolar.index_html", scodoc_dept=g.scodoc_dept)[
: -len("/index_html")
].encode(SCO_ENCODING)
]
def NotesURL():
@ -353,9 +349,7 @@ def NotesURL():
= url de base des méthodes de notes
(page accueil programmes).
"""
return url_for("notes.index_html", scodoc_dept=g.scodoc_dept)[
: -len("/index_html")
].encode(SCO_ENCODING)
return url_for("notes.index_html", scodoc_dept=g.scodoc_dept)[: -len("/index_html")]
def EntreprisesURL():
@ -364,7 +358,7 @@ def EntreprisesURL():
= url de base des requêtes de ZEntreprises
et page accueil Entreprises
"""
return "NotImplemented".encode(SCO_ENCODING)
return "NotImplemented"
# url_for("entreprises.index_html", scodoc_dept=g.scodoc_dept)[
# : -len("/index_html")
# ]
@ -374,7 +368,7 @@ def AbsencesURL():
"""URL of Absences"""
return url_for("absences.index_html", scodoc_dept=g.scodoc_dept)[
: -len("/index_html")
].encode(SCO_ENCODING)
]
def UsersURL():
@ -383,11 +377,7 @@ def UsersURL():
= url de base des requêtes ZScoUsers
et page accueil users
"""
return url_for("users.index_html", scodoc_dept=g.scodoc_dept)[
: -len("/index_html")
].encode(
SCO_ENCODING
) # sco8
return url_for("users.index_html", scodoc_dept=g.scodoc_dept)[: -len("/index_html")]
def get_current_user_name(REQUEST):
@ -614,13 +604,13 @@ def strupper(s):
# return s.decode(SCO_ENCODING).upper().encode(SCO_ENCODING)
# XXX fonctions inutiles en Python3 !
def strlower(s):
return s.lower()
# return s.decode(SCO_ENCODING).lower().encode(SCO_ENCODING)
def strcapitalize(s):
return s.decode(SCO_ENCODING).capitalize().encode(SCO_ENCODING)
return s.capitalize()
def abbrev_prenom(prenom):
@ -630,7 +620,7 @@ def abbrev_prenom(prenom):
# Charles -> Ch.
# Jean-Christophe -> J.-C.
# Marie Odile -> M. O.
prenom = prenom.decode(SCO_ENCODING).replace(".", " ").strip()
prenom = prenom.replace(".", " ").strip()
if not prenom:
return ""
d = prenom[:3].upper()
@ -654,7 +644,7 @@ def abbrev_prenom(prenom):
if i < n:
abrv += sep + prenom[i].upper() + "."
i += 1
return abrv.encode(SCO_ENCODING)
return abrv
#