From bfa6973d4ee8d662a8d1eb636246d3dc9e1e9cdc Mon Sep 17 00:00:00 2001 From: Emmanuel Viennet Date: Mon, 3 Apr 2023 17:40:45 +0200 Subject: [PATCH] WIP: migrating to SQlAlchemy 2.0.8 --- app/__init__.py | 22 ++++--- app/models/__init__.py | 2 - app/models/but_refcomp.py | 7 ++- app/models/but_validations.py | 4 +- app/models/formations.py | 8 +-- app/models/formsemestre.py | 7 ++- app/models/moduleimpls.py | 4 +- app/models/notes.py | 7 ++- app/models/raw_sql_init.py | 57 ------------------- app/scodoc/sco_abs_billets.py | 6 +- app/scodoc/sco_edit_ue.py | 3 +- app/scodoc/sco_utils.py | 2 +- config.py | 3 - flask_cas/__init__.py | 10 +--- .../25e3ca6cc063_dispenseue_par_semestre.py | 16 ++++-- .../versions/5542cac8c34a_semset_periode.py | 4 +- .../ae9bb0feea7a_contraintes_identite.py | 18 ++++-- .../b9aadc10227f_module_type_non_null.py | 2 +- ...codoc_9_0_51_add_unicity_constraint_on_.py | 4 +- 19 files changed, 70 insertions(+), 116 deletions(-) delete mode 100644 app/models/raw_sql_init.py diff --git a/app/__init__.py b/app/__init__.py index a3ac96b6..d8ffd4d3 100755 --- a/app/__init__.py +++ b/app/__init__.py @@ -13,11 +13,11 @@ import logging from logging.handlers import SMTPHandler, WatchedFileHandler from threading import Thread +import flask from flask import current_app, g, request from flask import Flask from flask import abort, flash, has_request_context, jsonify from flask import render_template -from flask.json import JSONEncoder from flask.logging import default_handler from flask_bootstrap import Bootstrap @@ -29,7 +29,7 @@ from flask_moment import Moment from flask_sqlalchemy import SQLAlchemy from jinja2 import select_autoescape -import sqlalchemy +import sqlalchemy as sa from flask_cas import CAS @@ -140,7 +140,7 @@ def handle_invalid_usage(error): # JSON ENCODING -class ScoDocJSONEncoder(JSONEncoder): +class ScoDocJSONEncoder(flask.json.provider.DefaultJSONProvider): def default(self, o): if isinstance(o, (datetime.datetime, datetime.date)): return o.isoformat() @@ -248,9 +248,13 @@ def create_app(config_class=DevConfig): CAS(app, url_prefix="/cas", configuration_function=cas.set_cas_configuration) app.wsgi_app = ReverseProxied(app.wsgi_app) - app.json_encoder = ScoDocJSONEncoder + app.json_provider_class = ScoDocJSONEncoder app.config.from_object(config_class) + # Pour conserver l'ordre des objets dans les JSON: + # e.g. l'ordre des UE dans les bulletins + app.json.sort_keys = False + # Evite de logguer toutes les requetes dans notre log logging.getLogger("werkzeug").disabled = True app.logger.setLevel(app.config["LOG_LEVEL"]) @@ -409,7 +413,7 @@ def create_app(config_class=DevConfig): with app.app_context(): try: set_cas_configuration(app) - except sqlalchemy.exc.ProgrammingError: + except sa.exc.ProgrammingError: # Si la base n'a pas été upgradée (arrive durrant l'install) # il se peut que la table scodoc_site_config n'existe pas encore. pass @@ -421,7 +425,7 @@ def set_sco_dept(scodoc_dept: str, open_cnx=True): # Check that dept exists try: dept = Departement.query.filter_by(acronym=scodoc_dept).first() - except sqlalchemy.exc.OperationalError: + except sa.exc.OperationalError: abort(503) if not dept: raise ScoValueError(f"Invalid dept: {scodoc_dept}") @@ -499,14 +503,15 @@ def truncate_database(): """ # use a stored SQL function, see createtables.sql try: - db.session.execute("SELECT truncate_tables('scodoc');") + db.session.execute(sa.text("SELECT truncate_tables('scodoc');")) db.session.commit() except: db.session.rollback() raise # Remet les compteurs (séquences sql) à zéro db.session.execute( - """ + sa.text( + """ CREATE OR REPLACE FUNCTION reset_sequences(username IN VARCHAR) RETURNS void AS $$ DECLARE statements CURSOR FOR @@ -522,6 +527,7 @@ def truncate_database(): SELECT reset_sequences('scodoc'); """ + ) ) db.session.commit() diff --git a/app/models/__init__.py b/app/models/__init__.py index 7864f660..032ddc86 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -21,8 +21,6 @@ convention = { metadata_obj = sqlalchemy.MetaData(naming_convention=convention) -from app.models.raw_sql_init import create_database_functions - from app.models.absences import Absence, AbsenceNotification, BilletAbsence from app.models.departements import Departement from app.models.etudiants import ( diff --git a/app/models/but_refcomp.py b/app/models/but_refcomp.py index d23ee313..edeb6ee2 100644 --- a/app/models/but_refcomp.py +++ b/app/models/but_refcomp.py @@ -7,7 +7,7 @@ """ from datetime import datetime -import flask_sqlalchemy +from flask_sqlalchemy.query import Query from sqlalchemy.orm import class_mapper import sqlalchemy @@ -307,6 +307,7 @@ class ApcSituationPro(db.Model, XMLModel): nullable=False, ) libelle = db.Column(db.Text(), nullable=False) + # aucun attribut (le text devient le libellé) def to_dict(self): return {"libelle": self.libelle} @@ -451,7 +452,7 @@ class ApcAppCritique(db.Model, XMLModel): ref_comp: ApcReferentielCompetences, annee: str, competence: ApcCompetence = None, - ) -> flask_sqlalchemy.BaseQuery: + ) -> Query: "Liste les AC de tous les parcours de ref_comp pour l'année indiquée" assert annee in {"BUT1", "BUT2", "BUT3"} query = cls.query.filter( @@ -550,7 +551,7 @@ class ApcParcours(db.Model, XMLModel): d["annees"] = {x.ordre: x.to_dict() for x in self.annees} return d - def query_competences(self) -> flask_sqlalchemy.BaseQuery: + def query_competences(self) -> Query: "Les compétences associées à ce parcours" return ( ApcCompetence.query.join(ApcParcoursNiveauCompetence, ApcAnneeParcours) diff --git a/app/models/but_validations.py b/app/models/but_validations.py index 364f3293..a21cd071 100644 --- a/app/models/but_validations.py +++ b/app/models/but_validations.py @@ -4,7 +4,7 @@ """ from typing import Union -import flask_sqlalchemy +from flask_sqlalchemy.query import Query from app import db from app.models import CODE_STR_LEN @@ -177,7 +177,7 @@ class RegroupementCoherentUE: def query_validations( self, - ) -> flask_sqlalchemy.BaseQuery: # list[ApcValidationRCUE] + ) -> Query: # list[ApcValidationRCUE] """Les validations de jury enregistrées pour ce RCUE""" niveau = self.ue_2.niveau_competence diff --git a/app/models/formations.py b/app/models/formations.py index fb395b46..579ad2bf 100644 --- a/app/models/formations.py +++ b/app/models/formations.py @@ -1,6 +1,6 @@ """ScoDoc 9 models : Formations """ -import flask_sqlalchemy +from flask_sqlalchemy.query import Query import app from app import db @@ -213,7 +213,7 @@ class Formation(db.Model): if change: app.clear_scodoc_cache() - def query_ues_parcour(self, parcour: ApcParcours) -> flask_sqlalchemy.BaseQuery: + def query_ues_parcour(self, parcour: ApcParcours) -> Query: """Les UEs d'un parcours de la formation. Si parcour est None, les UE sans parcours. Exemple: pour avoir les UE du semestre 3, faire @@ -231,9 +231,7 @@ class Formation(db.Model): ApcAnneeParcours.parcours_id == parcour.id, ) - def query_competences_parcour( - self, parcour: ApcParcours - ) -> flask_sqlalchemy.BaseQuery: + def query_competences_parcour(self, parcour: ApcParcours) -> Query: """Les ApcCompetences d'un parcours de la formation. None si pas de référentiel de compétences. """ diff --git a/app/models/formsemestre.py b/app/models/formsemestre.py index 0ec4e6be..e4b939e4 100644 --- a/app/models/formsemestre.py +++ b/app/models/formsemestre.py @@ -14,7 +14,8 @@ import datetime from functools import cached_property from flask_login import current_user -import flask_sqlalchemy +from flask_sqlalchemy.query import Query + from flask import flash, g from sqlalchemy import and_, or_ from sqlalchemy.sql import text @@ -281,7 +282,7 @@ class FormSemestre(db.Model): ) return r or [] - def query_ues(self, with_sport=False) -> flask_sqlalchemy.BaseQuery: + def query_ues(self, with_sport=False) -> Query: """UE des modules de ce semestre, triées par numéro. - Formations classiques: les UEs auxquelles appartiennent les modules mis en place dans ce semestre. @@ -311,7 +312,7 @@ class FormSemestre(db.Model): sem_ues = sem_ues.filter(UniteEns.type != codes_cursus.UE_SPORT) return sem_ues.order_by(UniteEns.numero) - def query_ues_parcours_etud(self, etudid: int) -> flask_sqlalchemy.BaseQuery: + def query_ues_parcours_etud(self, etudid: int) -> Query: """XXX inutilisé à part pour un test unitaire => supprimer ? UEs que suit l'étudiant dans ce semestre BUT en fonction du parcours dans lequel il est inscrit. diff --git a/app/models/moduleimpls.py b/app/models/moduleimpls.py index 2223538c..c4d7c3fe 100644 --- a/app/models/moduleimpls.py +++ b/app/models/moduleimpls.py @@ -2,7 +2,7 @@ """ScoDoc models: moduleimpls """ import pandas as pd -import flask_sqlalchemy +from flask_sqlalchemy.query import Query from app import db from app.auth.models import User @@ -179,7 +179,7 @@ class ModuleImplInscription(db.Model): @classmethod def etud_modimpls_in_ue( cls, formsemestre_id: int, etudid: int, ue_id: int - ) -> flask_sqlalchemy.BaseQuery: + ) -> Query: """moduleimpls de l'UE auxquels l'étudiant est inscrit. (Attention: inutile en APC, il faut considérer les coefficients) """ diff --git a/app/models/notes.py b/app/models/notes.py index 74bf2f18..0f82e286 100644 --- a/app/models/notes.py +++ b/app/models/notes.py @@ -3,6 +3,7 @@ """Notes, décisions de jury, évènements scolaires """ +import sqlalchemy as sa from app import db import app.scodoc.sco_utils as scu @@ -86,7 +87,8 @@ def etud_has_notes_attente(etudid, formsemestre_id): (ne compte que les notes en attente dans des évaluations avec coef. non nul). """ cursor = db.session.execute( - """SELECT COUNT(*) + sa.text( + """SELECT COUNT(*) FROM notes_notes n, notes_evaluation e, notes_moduleimpl m, notes_moduleimpl_inscription i WHERE n.etudid = :etudid @@ -97,7 +99,8 @@ def etud_has_notes_attente(etudid, formsemestre_id): and e.coefficient != 0 and m.id = i.moduleimpl_id and i.etudid = :etudid - """, + """ + ), { "formsemestre_id": formsemestre_id, "etudid": etudid, diff --git a/app/models/raw_sql_init.py b/app/models/raw_sql_init.py deleted file mode 100644 index 58094687..00000000 --- a/app/models/raw_sql_init.py +++ /dev/null @@ -1,57 +0,0 @@ -# -*- coding: UTF-8 -* - -""" -Create some Postgresql sequences and functions used by ScoDoc -using raw SQL -""" - -from app import db - - -def create_database_functions(): # XXX obsolete - """Create specific SQL functions and sequences - - XXX Obsolete: cette fonction est dans la première migration 9.0.3 - Flask-Migrate fait maintenant (dans les versions >= 9.0.4) ce travail. - """ - # Important: toujours utiliser IF NOT EXISTS - # car cette fonction peut être appelée plusieurs fois sur la même db - db.session.execute( - """ -CREATE SEQUENCE IF NOT EXISTS notes_idgen_fcod; -CREATE OR REPLACE FUNCTION notes_newid_fcod() RETURNS TEXT - AS $$ SELECT 'FCOD' || to_char(nextval('notes_idgen_fcod'), 'FM999999999'); $$ - LANGUAGE SQL; -CREATE OR REPLACE FUNCTION notes_newid_ucod() RETURNS TEXT - AS $$ SELECT 'UCOD' || to_char(nextval('notes_idgen_fcod'), 'FM999999999'); $$ - LANGUAGE SQL; - -CREATE OR REPLACE FUNCTION truncate_tables(username IN VARCHAR) RETURNS void AS $$ -DECLARE - statements CURSOR FOR - SELECT tablename FROM pg_tables - WHERE tableowner = username AND schemaname = 'public' - AND tablename <> 'notes_semestres' - AND tablename <> 'notes_form_modalites' - AND tablename <> 'alembic_version'; -BEGIN - FOR stmt IN statements LOOP - EXECUTE 'TRUNCATE TABLE ' || quote_ident(stmt.tablename) || ' CASCADE;'; - END LOOP; -END; -$$ LANGUAGE plpgsql; - --- Fonction pour anonymisation: --- inspirée par https://www.simononsoftware.com/random-string-in-postgresql/ -CREATE OR REPLACE FUNCTION random_text_md5( integer ) returns text - LANGUAGE SQL - AS $$ - select upper( substring( (SELECT string_agg(md5(random()::TEXT), '') - FROM generate_series( - 1, - CEIL($1 / 32.)::integer) - ), 1, $1) ); - $$; - """ - ) - db.session.commit() diff --git a/app/scodoc/sco_abs_billets.py b/app/scodoc/sco_abs_billets.py index cdee2373..82f8b094 100644 --- a/app/scodoc/sco_abs_billets.py +++ b/app/scodoc/sco_abs_billets.py @@ -29,16 +29,14 @@ """ from flask import g, url_for -import flask_sqlalchemy +from flask_sqlalchemy.query import Query from app.models.absences import BilletAbsence from app.models.etudiants import Identite from app.scodoc.gen_tables import GenTable from app.scodoc import sco_preferences -def query_billets_etud( - etudid: int = None, etat: bool = None -) -> flask_sqlalchemy.BaseQuery: +def query_billets_etud(etudid: int = None, etat: bool = None) -> Query: """Billets d'absences pour un étudiant, ou tous si etudid is None. Si etat, filtre par état. Si dans un département et que la gestion des billets n'a pas été activée diff --git a/app/scodoc/sco_edit_ue.py b/app/scodoc/sco_edit_ue.py index 9de82d14..166268e2 100644 --- a/app/scodoc/sco_edit_ue.py +++ b/app/scodoc/sco_edit_ue.py @@ -30,6 +30,7 @@ """ import re +import sqlalchemy as sa import flask from flask import flash, render_template, url_for from flask import g, request @@ -127,7 +128,7 @@ def do_ue_create(args): ): # évite les conflits de code while True: - cursor = db.session.execute("select notes_newid_ucod();") + cursor = db.session.execute(sa.text("select notes_newid_ucod();")) code = cursor.fetchone()[0] if UniteEns.query.filter_by(ue_code=code).count() == 0: break diff --git a/app/scodoc/sco_utils.py b/app/scodoc/sco_utils.py index 51056f14..b4babbac 100644 --- a/app/scodoc/sco_utils.py +++ b/app/scodoc/sco_utils.py @@ -855,7 +855,7 @@ def sendPDFFile(data, filename): # DEPRECATED utiliser send_file return send_file(data, filename=filename, mime=PDF_MIMETYPE, attached=True) -class ScoDocJSONEncoder(json.JSONEncoder): +class ScoDocJSONEncoder(flask.json.provider.DefaultJSONProvider): def default(self, o): # pylint: disable=E0202 if isinstance(o, (datetime.date, datetime.datetime)): return o.isoformat() diff --git a/config.py b/config.py index ed456b0b..ca8097d4 100755 --- a/config.py +++ b/config.py @@ -38,9 +38,6 @@ class Config: SCODOC_ERR_FILE = os.path.join(SCODOC_VAR_DIR, "log", "scodoc_exc.log") # MAX_CONTENT_LENGTH = 16 * 1024 * 1024 # Flask uploads (16Mo, en ligne avec nginx) - # Pour conserver l'ordre des objets dans les JSON: - # e.g. l'ordre des UE dans les bulletins - JSON_SORT_KEYS = False class ProdConfig(Config): diff --git a/flask_cas/__init__.py b/flask_cas/__init__.py index eae42139..fc3060f2 100644 --- a/flask_cas/__init__.py +++ b/flask_cas/__init__.py @@ -5,14 +5,6 @@ flask_cas.__init__ import flask from flask import current_app -# Find the stack on which we want to store the database connection. -# Starting with Flask 0.9, the _app_ctx_stack is the correct one, -# before that we need to use the _request_ctx_stack. -try: - from flask import _app_ctx_stack as stack -except ImportError: - from flask import _request_ctx_stack as stack - from . import routing from functools import wraps @@ -67,7 +59,7 @@ class CAS(object): app.teardown_request(self.teardown) def teardown(self, exception): - ctx = stack.top + pass # ctx = stack.top @property def app(self): diff --git a/migrations/versions/25e3ca6cc063_dispenseue_par_semestre.py b/migrations/versions/25e3ca6cc063_dispenseue_par_semestre.py index 942ef77d..b4665459 100644 --- a/migrations/versions/25e3ca6cc063_dispenseue_par_semestre.py +++ b/migrations/versions/25e3ca6cc063_dispenseue_par_semestre.py @@ -43,11 +43,14 @@ def upgrade(): bind = op.get_bind() session = Session(bind=bind) dispenses = session.execute( - """SELECT id, ue_id, etudid FROM "dispenseUE" WHERE formsemestre_id IS NULL;""" + sa.text( + """SELECT id, ue_id, etudid FROM "dispenseUE" WHERE formsemestre_id IS NULL;""" + ) ).all() for dispense_id, ue_id, etudid in dispenses: formsemestre_ids = session.execute( - """ + sa.text( + """ SELECT notes_formsemestre.id FROM notes_formsemestre, notes_formations, notes_ue, notes_formsemestre_inscription WHERE notes_formsemestre.formation_id = notes_formations.id @@ -58,14 +61,17 @@ def upgrade(): and notes_formsemestre_inscription.etudid = :etudid ORDER BY notes_formsemestre.date_debut DESC LIMIT 1; - """, + """ + ), {"ue_id": ue_id, "etudid": etudid}, ).all() if formsemestre_ids: formsemestre_id = formsemestre_ids[0][0] session.execute( - """ - UPDATE "dispenseUE" SET formsemestre_id=:formsemestre_id WHERE id=:dispense_id""", + sa.text( + """ + UPDATE "dispenseUE" SET formsemestre_id=:formsemestre_id WHERE id=:dispense_id""" + ), {"formsemestre_id": formsemestre_id, "dispense_id": dispense_id}, ) diff --git a/migrations/versions/5542cac8c34a_semset_periode.py b/migrations/versions/5542cac8c34a_semset_periode.py index 4b5a0ff7..d2f94c40 100644 --- a/migrations/versions/5542cac8c34a_semset_periode.py +++ b/migrations/versions/5542cac8c34a_semset_periode.py @@ -23,7 +23,9 @@ def upgrade(): # bind = op.get_bind() session = Session(bind=bind) - session.execute("""UPDATE notes_semset SET sem_id=0 WHERE sem_id IS NULL;""") + session.execute( + sa.text("""UPDATE notes_semset SET sem_id=0 WHERE sem_id IS NULL;""") + ) op.alter_column( "notes_semset", "sem_id", existing_type=sa.INTEGER(), nullable=False ) diff --git a/migrations/versions/ae9bb0feea7a_contraintes_identite.py b/migrations/versions/ae9bb0feea7a_contraintes_identite.py index 6ab05144..0a752d26 100644 --- a/migrations/versions/ae9bb0feea7a_contraintes_identite.py +++ b/migrations/versions/ae9bb0feea7a_contraintes_identite.py @@ -29,21 +29,25 @@ def upgrade(): session = Session(bind=bind) # Corrige NIP dups = session.execute( - """SELECT dept_id, code_nip + sa.text( + """SELECT dept_id, code_nip FROM identite WHERE code_nip IS NOT NULL GROUP BY dept_id, code_nip HAVING COUNT(*) > 1;""" + ) ).all() for dept_id, code_nip in dups: etuds_dups = session.execute( - """SELECT id, nom, prenom FROM identite - WHERE dept_id=:dept_id AND code_nip=:code_nip""", + sa.text( + """SELECT id, nom, prenom FROM identite + WHERE dept_id=:dept_id AND code_nip=:code_nip""" + ), {"dept_id": dept_id, "code_nip": code_nip}, ).all() for i, (etudid, nom, prenom) in enumerate(etuds_dups[1:], start=1): session.execute( - """UPDATE identite SET code_nip=:code_nip WHERE id=:etudid""", + sa.text("""UPDATE identite SET code_nip=:code_nip WHERE id=:etudid"""), { "code_nip": f"{code_nip}-{i}", "etudid": etudid, @@ -55,11 +59,13 @@ def upgrade(): session.commit() # Corrige INE dups = session.execute( - """SELECT dept_id, code_ine + sa.text( + """SELECT dept_id, code_ine FROM identite WHERE code_ine IS NOT NULL GROUP BY dept_id, code_ine HAVING COUNT(*) > 1;""" + ) ).all() for dept_id, code_ine in dups: etuds_dups = session.execute( @@ -69,7 +75,7 @@ def upgrade(): ).all() for i, (etudid, nom, prenom) in enumerate(etuds_dups[1:], start=1): session.execute( - """UPDATE identite SET code_ine=:code_ine WHERE id=:etudid""", + sa.text("""UPDATE identite SET code_ine=:code_ine WHERE id=:etudid"""), { "code_ine": f"{code_ine}-{i}", "etudid": etudid, diff --git a/migrations/versions/b9aadc10227f_module_type_non_null.py b/migrations/versions/b9aadc10227f_module_type_non_null.py index 1b5d4548..9eb48c44 100644 --- a/migrations/versions/b9aadc10227f_module_type_non_null.py +++ b/migrations/versions/b9aadc10227f_module_type_non_null.py @@ -23,7 +23,7 @@ def upgrade(): bind = op.get_bind() session = Session(bind=bind) session.execute( - """UPDATE notes_modules SET module_type=0 WHERE module_type IS NULL;""" + sa.text("""UPDATE notes_modules SET module_type=0 WHERE module_type IS NULL;""") ) # ### commands auto generated by Alembic - please adjust! ### op.alter_column( diff --git a/migrations/versions/d74b4e16fb3c_scodoc_9_0_51_add_unicity_constraint_on_.py b/migrations/versions/d74b4e16fb3c_scodoc_9_0_51_add_unicity_constraint_on_.py index 1c9da8a4..4db6c0f7 100644 --- a/migrations/versions/d74b4e16fb3c_scodoc_9_0_51_add_unicity_constraint_on_.py +++ b/migrations/versions/d74b4e16fb3c_scodoc_9_0_51_add_unicity_constraint_on_.py @@ -24,13 +24,15 @@ def upgrade(): bind = op.get_bind() session = Session(bind=bind) session.execute( - """ + sa.text( + """ DELETE FROM notes_moduleimpl_inscription i1 USING notes_moduleimpl_inscription i2 WHERE i1.id < i2.id AND i1.moduleimpl_id = i2.moduleimpl_id AND i1.etudid = i2.etudid; """ + ) ) # ### commands auto generated by Alembic - please adjust! ###