WIP: migrating to SQlAlchemy 2.0.8

app/__init__.py

@@ -13,11 +13,11 @@ import logging
 from logging.handlers import SMTPHandler, WatchedFileHandler
 from threading import Thread
 
+import flask
 from flask import current_app, g, request
 from flask import Flask
 from flask import abort, flash, has_request_context, jsonify
 from flask import render_template
-from flask.json import JSONEncoder
 from flask.logging import default_handler
 
 from flask_bootstrap import Bootstrap
@@ -29,7 +29,7 @@ from flask_moment import Moment
 from flask_sqlalchemy import SQLAlchemy
 
 from jinja2 import select_autoescape
-import sqlalchemy
+import sqlalchemy as sa
 
 from flask_cas import CAS
 
@@ -140,7 +140,7 @@ def handle_invalid_usage(error):
 
 
 # JSON ENCODING
-class ScoDocJSONEncoder(JSONEncoder):
+class ScoDocJSONEncoder(flask.json.provider.DefaultJSONProvider):
     def default(self, o):
         if isinstance(o, (datetime.datetime, datetime.date)):
             return o.isoformat()
@@ -248,9 +248,13 @@ def create_app(config_class=DevConfig):
 
     CAS(app, url_prefix="/cas", configuration_function=cas.set_cas_configuration)
     app.wsgi_app = ReverseProxied(app.wsgi_app)
-    app.json_encoder = ScoDocJSONEncoder
+    app.json_provider_class = ScoDocJSONEncoder
 
     app.config.from_object(config_class)
+    # Pour conserver l'ordre des objets dans les JSON:
+    # e.g. l'ordre des UE dans les bulletins
+    app.json.sort_keys = False
+
     # Evite de logguer toutes les requetes dans notre log
     logging.getLogger("werkzeug").disabled = True
     app.logger.setLevel(app.config["LOG_LEVEL"])
@@ -405,7 +409,7 @@ def create_app(config_class=DevConfig):
     with app.app_context():
         try:
             set_cas_configuration(app)
-        except sqlalchemy.exc.ProgrammingError:
+        except sa.exc.ProgrammingError:
             # Si la base n'a pas été upgradée (arrive durrant l'install)
             # il se peut que la table scodoc_site_config n'existe pas encore.
             pass
@@ -417,7 +421,7 @@ def set_sco_dept(scodoc_dept: str, open_cnx=True):
     # Check that dept exists
     try:
         dept = Departement.query.filter_by(acronym=scodoc_dept).first()
-    except sqlalchemy.exc.OperationalError:
+    except sa.exc.OperationalError:
         abort(503)
     if not dept:
         raise ScoValueError(f"Invalid dept: {scodoc_dept}")
@@ -495,13 +499,14 @@ def truncate_database():
     """
     # use a stored SQL function, see createtables.sql
     try:
-        db.session.execute("SELECT truncate_tables('scodoc');")
+        db.session.execute(sa.text("SELECT truncate_tables('scodoc');"))
         db.session.commit()
     except:
         db.session.rollback()
         raise
     # Remet les compteurs (séquences sql) à zéro
     db.session.execute(
+        sa.text(
             """
     CREATE OR REPLACE FUNCTION reset_sequences(username IN VARCHAR) RETURNS void AS $$
     DECLARE
@@ -519,6 +524,7 @@ def truncate_database():
     SELECT  reset_sequences('scodoc');
     """
         )
+    )
     db.session.commit()
 
 

app/models/__init__.py

@@ -21,8 +21,6 @@ convention = {
 
 metadata_obj = sqlalchemy.MetaData(naming_convention=convention)
 
-from app.models.raw_sql_init import create_database_functions
-
 from app.models.absences import Absence, AbsenceNotification, BilletAbsence
 from app.models.departements import Departement
 from app.models.etudiants import (

app/models/but_refcomp.py

@@ -7,7 +7,7 @@
 """
 from datetime import datetime
 
-import flask_sqlalchemy
+from flask_sqlalchemy.query import Query
 from sqlalchemy.orm import class_mapper
 import sqlalchemy
 
@@ -307,6 +307,7 @@ class ApcSituationPro(db.Model, XMLModel):
         nullable=False,
     )
     libelle = db.Column(db.Text(), nullable=False)
+
     # aucun attribut (le text devient le libellé)
     def to_dict(self):
         return {"libelle": self.libelle}
@@ -451,7 +452,7 @@ class ApcAppCritique(db.Model, XMLModel):
         ref_comp: ApcReferentielCompetences,
         annee: str,
         competence: ApcCompetence = None,
-    ) -> flask_sqlalchemy.BaseQuery:
+    ) -> Query:
         "Liste les AC de tous les parcours de ref_comp pour l'année indiquée"
         assert annee in {"BUT1", "BUT2", "BUT3"}
         query = cls.query.filter(
@@ -550,7 +551,7 @@ class ApcParcours(db.Model, XMLModel):
             d["annees"] = {x.ordre: x.to_dict() for x in self.annees}
         return d
 
-    def query_competences(self) -> flask_sqlalchemy.BaseQuery:
+    def query_competences(self) -> Query:
         "Les compétences associées à ce parcours"
         return (
             ApcCompetence.query.join(ApcParcoursNiveauCompetence, ApcAnneeParcours)

app/models/but_validations.py

@@ -4,7 +4,7 @@
 """
 from typing import Union
 
-import flask_sqlalchemy
+from flask_sqlalchemy.query import Query
 
 from app import db
 from app.models import CODE_STR_LEN
@@ -177,7 +177,7 @@ class RegroupementCoherentUE:
 
     def query_validations(
         self,
-    ) -> flask_sqlalchemy.BaseQuery:  # list[ApcValidationRCUE]
+    ) -> Query:  # list[ApcValidationRCUE]
         """Les validations de jury enregistrées pour ce RCUE"""
         niveau = self.ue_2.niveau_competence
 

app/models/formations.py

@@ -1,6 +1,6 @@
 """ScoDoc 9 models : Formations
 """
-import flask_sqlalchemy
+from flask_sqlalchemy.query import Query
 
 import app
 from app import db
@@ -213,7 +213,7 @@ class Formation(db.Model):
         if change:
             app.clear_scodoc_cache()
 
-    def query_ues_parcour(self, parcour: ApcParcours) -> flask_sqlalchemy.BaseQuery:
+    def query_ues_parcour(self, parcour: ApcParcours) -> Query:
         """Les UEs d'un parcours de la formation.
         Si parcour est None, les UE sans parcours.
         Exemple: pour avoir les UE du semestre 3, faire
@@ -231,9 +231,7 @@ class Formation(db.Model):
             ApcAnneeParcours.parcours_id == parcour.id,
         )
 
-    def query_competences_parcour(
+    def query_competences_parcour(self, parcour: ApcParcours) -> Query:
-        self, parcour: ApcParcours
-    ) -> flask_sqlalchemy.BaseQuery:
         """Les ApcCompetences d'un parcours de la formation.
         None si pas de référentiel de compétences.
         """

app/models/formsemestre.py

@@ -14,7 +14,8 @@ import datetime
 from functools import cached_property
 
 from flask_login import current_user
-import flask_sqlalchemy
+from flask_sqlalchemy.query import Query
+
 from flask import flash, g
 from sqlalchemy import and_, or_
 from sqlalchemy.sql import text
@@ -281,7 +282,7 @@ class FormSemestre(db.Model):
         )
         return r or []
 
-    def query_ues(self, with_sport=False) -> flask_sqlalchemy.BaseQuery:
+    def query_ues(self, with_sport=False) -> Query:
         """UE des modules de ce semestre, triées par numéro.
         - Formations classiques: les UEs auxquelles appartiennent
           les modules mis en place dans ce semestre.
@@ -311,7 +312,7 @@ class FormSemestre(db.Model):
             sem_ues = sem_ues.filter(UniteEns.type != codes_cursus.UE_SPORT)
         return sem_ues.order_by(UniteEns.numero)
 
-    def query_ues_parcours_etud(self, etudid: int) -> flask_sqlalchemy.BaseQuery:
+    def query_ues_parcours_etud(self, etudid: int) -> Query:
         """XXX inutilisé à part pour un test unitaire => supprimer ?
         UEs que suit l'étudiant dans ce semestre BUT
         en fonction du parcours dans lequel il est inscrit.
@@ -967,7 +968,7 @@ class FormationModalite(db.Model):
         """Create default modalities"""
         numero = 0
         try:
-            for (code, titre) in (
+            for code, titre in (
                 (FormationModalite.DEFAULT_MODALITE, "Formation Initiale"),
                 ("FAP", "Apprentissage"),
                 ("FC", "Formation Continue"),

app/models/moduleimpls.py

@@ -2,7 +2,7 @@
 """ScoDoc models: moduleimpls
 """
 import pandas as pd
-import flask_sqlalchemy
+from flask_sqlalchemy.query import Query
 
 from app import db
 from app.auth.models import User
@@ -163,7 +163,7 @@ class ModuleImplInscription(db.Model):
     @classmethod
     def etud_modimpls_in_ue(
         cls, formsemestre_id: int, etudid: int, ue_id: int
-    ) -> flask_sqlalchemy.BaseQuery:
+    ) -> Query:
         """moduleimpls de l'UE auxquels l'étudiant est inscrit.
         (Attention: inutile en APC, il faut considérer les coefficients)
         """

app/models/notes.py

@@ -3,6 +3,7 @@
 """Notes, décisions de jury, évènements scolaires
 """
 
+import sqlalchemy as sa
 from app import db
 import app.scodoc.sco_utils as scu
 
@@ -86,6 +87,7 @@ def etud_has_notes_attente(etudid, formsemestre_id):
     (ne compte que les notes en attente dans des évaluations avec coef. non nul).
     """
     cursor = db.session.execute(
+        sa.text(
             """SELECT COUNT(*)
         FROM notes_notes n, notes_evaluation e, notes_moduleimpl m,
         notes_moduleimpl_inscription i
@@ -97,7 +99,8 @@ def etud_has_notes_attente(etudid, formsemestre_id):
         and e.coefficient != 0
         and m.id = i.moduleimpl_id
         and i.etudid = :etudid
-        """,
+        """
+        ),
         {
             "formsemestre_id": formsemestre_id,
             "etudid": etudid,

app/models/raw_sql_init.py

@@ -1,57 +0,0 @@
-# -*- coding: UTF-8 -*
-
-"""
-Create some Postgresql sequences and functions used by ScoDoc
-using raw SQL
-"""
-
-from app import db
-
-
-def create_database_functions():  # XXX obsolete
-    """Create specific SQL functions and sequences
-
-    XXX Obsolete: cette fonction est dans la première migration 9.0.3
-    Flask-Migrate fait maintenant (dans les versions >= 9.0.4) ce travail.
-    """
-    # Important: toujours utiliser IF NOT EXISTS
-    # car cette fonction peut être appelée plusieurs fois sur la même db
-    db.session.execute(
-        """
-CREATE SEQUENCE IF NOT EXISTS notes_idgen_fcod;
-CREATE OR REPLACE FUNCTION notes_newid_fcod() RETURNS TEXT
-    AS $$ SELECT 'FCOD' || to_char(nextval('notes_idgen_fcod'), 'FM999999999');  $$
-    LANGUAGE SQL;
-CREATE OR REPLACE FUNCTION notes_newid_ucod() RETURNS TEXT
-    AS $$ SELECT 'UCOD' || to_char(nextval('notes_idgen_fcod'), 'FM999999999');  $$
-    LANGUAGE SQL;
-
-CREATE OR REPLACE FUNCTION truncate_tables(username IN VARCHAR) RETURNS void AS $$
-DECLARE
-    statements CURSOR FOR
-        SELECT tablename FROM pg_tables
-        WHERE tableowner = username AND schemaname = 'public'
-        AND tablename <> 'notes_semestres'
-        AND tablename <> 'notes_form_modalites'
-        AND tablename <> 'alembic_version';
-BEGIN
-    FOR stmt IN statements LOOP
-        EXECUTE 'TRUNCATE TABLE ' || quote_ident(stmt.tablename) || ' CASCADE;';
-    END LOOP;
-END;
-$$ LANGUAGE plpgsql;
-
--- Fonction pour anonymisation:
--- inspirée par https://www.simononsoftware.com/random-string-in-postgresql/
-CREATE OR REPLACE FUNCTION random_text_md5( integer ) returns text
-    LANGUAGE SQL
-    AS $$
-    select upper( substring( (SELECT string_agg(md5(random()::TEXT), '')
-    FROM generate_series(
-        1,
-        CEIL($1 / 32.)::integer)
-    ), 1, $1) );
-    $$;
-    """
-    )
-    db.session.commit()

app/scodoc/sco_abs_billets.py

@@ -29,16 +29,14 @@
 """
 
 from flask import g, url_for
-import flask_sqlalchemy
+from flask_sqlalchemy.query import Query
 from app.models.absences import BilletAbsence
 from app.models.etudiants import Identite
 from app.scodoc.gen_tables import GenTable
 from app.scodoc import sco_preferences
 
 
-def query_billets_etud(
+def query_billets_etud(etudid: int = None, etat: bool = None) -> Query:
-    etudid: int = None, etat: bool = None
-) -> flask_sqlalchemy.BaseQuery:
     """Billets d'absences pour un étudiant, ou tous si etudid is None.
     Si etat, filtre par état.
     Si dans un département et que la gestion des billets n'a pas été activée

app/scodoc/sco_edit_ue.py

@@ -30,6 +30,7 @@
 """
 import re
 
+import sqlalchemy as sa
 import flask
 from flask import flash, render_template, url_for
 from flask import g, request
@@ -127,7 +128,7 @@ def do_ue_create(args):
     ):
         # évite les conflits de code
         while True:
-            cursor = db.session.execute("select notes_newid_ucod();")
+            cursor = db.session.execute(sa.text("select notes_newid_ucod();"))
             code = cursor.fetchone()[0]
             if UniteEns.query.filter_by(ue_code=code).count() == 0:
                 break

app/scodoc/sco_utils.py

@@ -690,7 +690,7 @@ def sendPDFFile(data, filename):  # DEPRECATED  utiliser send_file
     return send_file(data, filename=filename, mime=PDF_MIMETYPE, attached=True)
 
 
-class ScoDocJSONEncoder(json.JSONEncoder):
+class ScoDocJSONEncoder(flask.json.provider.DefaultJSONProvider):
     def default(self, o):  # pylint: disable=E0202
         if isinstance(o, (datetime.date, datetime.datetime)):
             return o.isoformat()

config.py

@@ -38,9 +38,6 @@ class Config:
     SCODOC_ERR_FILE = os.path.join(SCODOC_VAR_DIR, "log", "scodoc_exc.log")
     #
     MAX_CONTENT_LENGTH = 16 * 1024 * 1024  # Flask uploads (16Mo, en ligne avec nginx)
-    # Pour conserver l'ordre des objets dans les JSON:
-    # e.g. l'ordre des UE dans les bulletins
-    JSON_SORT_KEYS = False
 
 
 class ProdConfig(Config):

flask_cas/__init__.py

@@ -5,14 +5,6 @@ flask_cas.__init__
 import flask
 from flask import current_app
 
-# Find the stack on which we want to store the database connection.
-# Starting with Flask 0.9, the _app_ctx_stack is the correct one,
-# before that we need to use the _request_ctx_stack.
-try:
-    from flask import _app_ctx_stack as stack
-except ImportError:
-    from flask import _request_ctx_stack as stack
-
 from . import routing
 
 from functools import wraps
@@ -67,7 +59,7 @@ class CAS(object):
             app.teardown_request(self.teardown)
 
     def teardown(self, exception):
-        ctx = stack.top
+        pass  # ctx = stack.top
 
     @property
     def app(self):

migrations/versions/25e3ca6cc063_dispenseue_par_semestre.py

@@ -43,10 +43,13 @@ def upgrade():
     bind = op.get_bind()
     session = Session(bind=bind)
     dispenses = session.execute(
+        sa.text(
             """SELECT id, ue_id, etudid FROM "dispenseUE" WHERE formsemestre_id IS NULL;"""
+        )
     ).all()
     for dispense_id, ue_id, etudid in dispenses:
         formsemestre_ids = session.execute(
+            sa.text(
                 """
             SELECT notes_formsemestre.id
             FROM notes_formsemestre, notes_formations, notes_ue, notes_formsemestre_inscription
@@ -58,14 +61,17 @@ def upgrade():
             and notes_formsemestre_inscription.etudid = :etudid
             ORDER BY notes_formsemestre.date_debut DESC
             LIMIT 1;
-            """,
+            """
+            ),
             {"ue_id": ue_id, "etudid": etudid},
         ).all()
         if formsemestre_ids:
             formsemestre_id = formsemestre_ids[0][0]
             session.execute(
+                sa.text(
                     """
-                UPDATE "dispenseUE" SET formsemestre_id=:formsemestre_id WHERE id=:dispense_id""",
+                UPDATE "dispenseUE" SET formsemestre_id=:formsemestre_id WHERE id=:dispense_id"""
+                ),
                 {"formsemestre_id": formsemestre_id, "dispense_id": dispense_id},
             )
 

migrations/versions/5542cac8c34a_semset_periode.py

@@ -23,7 +23,9 @@ def upgrade():
     #
     bind = op.get_bind()
     session = Session(bind=bind)
-    session.execute("""UPDATE notes_semset SET sem_id=0 WHERE sem_id IS NULL;""")
+    session.execute(
+        sa.text("""UPDATE notes_semset SET sem_id=0 WHERE sem_id IS NULL;""")
+    )
     op.alter_column(
         "notes_semset", "sem_id", existing_type=sa.INTEGER(), nullable=False
     )

migrations/versions/ae9bb0feea7a_contraintes_identite.py

@@ -29,21 +29,25 @@ def upgrade():
     session = Session(bind=bind)
     # Corrige NIP
     dups = session.execute(
+        sa.text(
             """SELECT dept_id, code_nip
             FROM identite
             WHERE code_nip IS NOT NULL
             GROUP BY dept_id, code_nip
             HAVING COUNT(*) > 1;"""
+        )
     ).all()
     for dept_id, code_nip in dups:
         etuds_dups = session.execute(
+            sa.text(
                 """SELECT id, nom, prenom FROM identite
-            WHERE dept_id=:dept_id AND code_nip=:code_nip""",
+            WHERE dept_id=:dept_id AND code_nip=:code_nip"""
+            ),
             {"dept_id": dept_id, "code_nip": code_nip},
         ).all()
         for i, (etudid, nom, prenom) in enumerate(etuds_dups[1:], start=1):
             session.execute(
-                """UPDATE identite SET code_nip=:code_nip WHERE id=:etudid""",
+                sa.text("""UPDATE identite SET code_nip=:code_nip WHERE id=:etudid"""),
                 {
                     "code_nip": f"{code_nip}-{i}",
                     "etudid": etudid,
@@ -55,11 +59,13 @@ def upgrade():
     session.commit()
     # Corrige INE
     dups = session.execute(
+        sa.text(
             """SELECT dept_id, code_ine
             FROM identite
             WHERE code_ine IS NOT NULL
             GROUP BY dept_id, code_ine
             HAVING COUNT(*) > 1;"""
+        )
     ).all()
     for dept_id, code_ine in dups:
         etuds_dups = session.execute(
@@ -69,7 +75,7 @@ def upgrade():
         ).all()
         for i, (etudid, nom, prenom) in enumerate(etuds_dups[1:], start=1):
             session.execute(
-                """UPDATE identite SET code_ine=:code_ine WHERE id=:etudid""",
+                sa.text("""UPDATE identite SET code_ine=:code_ine WHERE id=:etudid"""),
                 {
                     "code_ine": f"{code_ine}-{i}",
                     "etudid": etudid,

migrations/versions/b9aadc10227f_module_type_non_null.py

@@ -23,7 +23,7 @@ def upgrade():
     bind = op.get_bind()
     session = Session(bind=bind)
     session.execute(
-        """UPDATE notes_modules SET module_type=0 WHERE module_type IS NULL;"""
+        sa.text("""UPDATE notes_modules SET module_type=0 WHERE module_type IS NULL;""")
     )
     # ### commands auto generated by Alembic - please adjust! ###
     op.alter_column(

migrations/versions/d74b4e16fb3c_scodoc_9_0_51_add_unicity_constraint_on_.py

@@ -24,6 +24,7 @@ def upgrade():
     bind = op.get_bind()
     session = Session(bind=bind)
     session.execute(
+        sa.text(
             """
     DELETE FROM notes_moduleimpl_inscription i1
     USING notes_moduleimpl_inscription i2
@@ -32,6 +33,7 @@ def upgrade():
     AND i1.etudid = i2.etudid;
     """
         )
+    )
 
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_unique_constraint(