iziram/ScoDoc
1
0
Fork 0
forked from ScoDoc/ScoDoc
Code Issues 12 Pull Requests Projects 1 Releases Wiki Activity

WIP: ajustements pour upgrade SQLAlchemy

Browse Source
This commit is contained in:
Emmanuel Viennet 2023-04-04 09:57:54 +02:00 committed by iziram
parent bfa6973d4e
commit c23738e5dc
13 changed files with 126 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
app/__init__.py
View File

@ -3,6 +3,7 @@
import base64
import datetime
import json
import os
import socket
import sys
@ -12,12 +13,13 @@ import traceback
import logging
from logging.handlers import SMTPHandler, WatchedFileHandler
from threading import Thread
import warnings
import flask
from flask import current_app, g, request
from flask import Flask
from flask import abort, flash, has_request_context, jsonify
from flask import render_template
from flask.json import JSONEncoder
from flask.logging import default_handler
from flask_bootstrap import Bootstrap
@ -42,6 +44,8 @@ from app.scodoc.sco_exceptions import (
ScoValueError,
APIInvalidParams,
)
from app.scodoc.sco_vdi import ApoEtapeVDI
from config import DevConfig
import sco_version
@ -140,12 +144,14 @@ def handle_invalid_usage(error):
# JSON ENCODING
class ScoDocJSONEncoder(flask.json.provider.DefaultJSONProvider):
def default(self, o):
if isinstance(o, (datetime.datetime, datetime.date)):
class ScoDocJSONEncoder(JSONEncoder):
def default(self, o): # pylint: disable=E0202
if isinstance(o, (datetime.date, datetime.datetime)):
return o.isoformat()
return super().default(o)
elif isinstance(o, ApoEtapeVDI):
return str(o)
else:
return json.JSONEncoder.default(self, o)
def render_raw_html(template_filename: str, **args) -> str:
@ -258,6 +264,10 @@ def create_app(config_class=DevConfig):
# Evite de logguer toutes les requetes dans notre log
logging.getLogger("werkzeug").disabled = True
app.logger.setLevel(app.config["LOG_LEVEL"])
if app.config["TESTING"] or app.config["DEBUG"]:
# S'arrête sur tous les warnings, sauf
# flask_sqlalchemy/query (pb deprecation du model.get())
warnings.filterwarnings("error", module="flask_sqlalchemy/query")
# Vérifie/crée lien sym pour les URL statiques
link_filename = f"{app.root_path}/static/links/{sco_version.SCOVERSION}"

29
app/comp/inscr_mod.py
View File

@ -4,6 +4,7 @@
"""Matrices d'inscription aux modules d'un semestre
"""
import pandas as pd
import sqlalchemy as sa
from app import db
@ -12,6 +13,13 @@ from app import db
# sur test debug 116 etuds, 18 modules, on est autour de 250ms.
# On a testé trois approches, ci-dessous (et retenu la 1ere)
#
_load_modimpl_inscr_q = sa.text(
"""SELECT etudid, 1 AS ":moduleimpl_id"
FROM notes_moduleimpl_inscription
WHERE moduleimpl_id=:moduleimpl_id"""
)
def df_load_modimpl_inscr(formsemestre) -> pd.DataFrame:
"""Charge la matrice des inscriptions aux modules du semestre
rows: etudid (inscrits au semestre, avec DEM et DEF)
@ -22,17 +30,16 @@ def df_load_modimpl_inscr(formsemestre) -> pd.DataFrame:
moduleimpl_ids = [m.id for m in formsemestre.modimpls_sorted]
etudids = [inscr.etudid for inscr in formsemestre.inscriptions]
df = pd.DataFrame(index=etudids, dtype=int)
for moduleimpl_id in moduleimpl_ids:
ins_df = pd.read_sql_query(
"""SELECT etudid, 1 AS "%(moduleimpl_id)s"
FROM notes_moduleimpl_inscription
WHERE moduleimpl_id=%(moduleimpl_id)s""",
db.engine,
params={"moduleimpl_id": moduleimpl_id},
index_col="etudid",
dtype=int,
)
df = df.merge(ins_df, how="left", left_index=True, right_index=True)
with db.engine.begin() as connection:
for moduleimpl_id in moduleimpl_ids:
ins_df = pd.read_sql_query(
_load_modimpl_inscr_q,
connection,
params={"moduleimpl_id": moduleimpl_id},
index_col="etudid",
dtype=int,
)
df = df.merge(ins_df, how="left", left_index=True, right_index=True)
# Force columns names to integers (moduleimpl ids)
df.columns = pd.Index([int(x) for x in df.columns], dtype=int)
# les colonnes de df sont en float (Nan) quand il n'y a

18
app/comp/jury.py
View File

@ -7,6 +7,7 @@
"""Stockage des décisions de jury
"""
import pandas as pd
import sqlalchemy as sa
from app import db
from app.models import FormSemestre, Identite, ScolarFormSemestreValidation, UniteEns
@ -132,7 +133,8 @@ def formsemestre_get_ue_capitalisees(formsemestre: FormSemestre) -> pd.DataFrame
# Note: pour récupérer aussi les UE validées en CMp ou ADJ, changer une ligne
# and ( SFV.code = 'ADM' or SFV.code = 'ADJ' or SFV.code = 'CMP' )
query = """
query = sa.text(
"""
SELECT DISTINCT SFV.*, ue.ue_code
FROM
notes_ue ue,
@ -144,21 +146,22 @@ def formsemestre_get_ue_capitalisees(formsemestre: FormSemestre) -> pd.DataFrame
WHERE ue.formation_id = nf.id
and nf.formation_code = nf2.formation_code
and nf2.id=%(formation_id)s
and nf2.id=:formation_id
and ins.etudid = SFV.etudid
and ins.formsemestre_id = %(formsemestre_id)s
and ins.formsemestre_id = :formsemestre_id
and SFV.ue_id = ue.id
and SFV.code = 'ADM'
and ( (sem.id = SFV.formsemestre_id
and sem.date_debut < %(date_debut)s
and sem.semestre_id = %(semestre_id)s )
and sem.date_debut < :date_debut
and sem.semestre_id = :semestre_id )
or (
((SFV.formsemestre_id is NULL) OR (SFV.is_external)) -- les UE externes ou "anterieures"
AND (SFV.semestre_id is NULL OR SFV.semestre_id=%(semestre_id)s)
AND (SFV.semestre_id is NULL OR SFV.semestre_id=:semestre_id)
) )
"""
)
params = {
"formation_id": formsemestre.formation.id,
"formsemestre_id": formsemestre.id,
@ -166,5 +169,6 @@ def formsemestre_get_ue_capitalisees(formsemestre: FormSemestre) -> pd.DataFrame
"date_debut": formsemestre.date_debut,
}
df = pd.read_sql_query(query, db.engine, params=params, index_col="etudid")
with db.engine.begin() as connection:
df = pd.read_sql_query(query, connection, params=params, index_col="etudid")
return df

34
app/comp/moy_mod.py
View File

@ -38,6 +38,7 @@ from dataclasses import dataclass
import numpy as np
import pandas as pd
import sqlalchemy as sa
import app
from app import db
@ -192,24 +193,29 @@ class ModuleImplResults:
evals_notes.columns = pd.Index([int(x) for x in evals_notes.columns], dtype=int)
self.evals_notes = evals_notes
_load_evaluation_notes_q = sa.text(
"""SELECT n.etudid, n.value AS ":evaluation_id"
FROM notes_notes n, notes_moduleimpl_inscription i
WHERE evaluation_id=:evaluation_id
AND n.etudid = i.etudid
AND i.moduleimpl_id = :moduleimpl_id
"""
)
def _load_evaluation_notes(self, evaluation: Evaluation) -> pd.DataFrame:
"""Charge les notes de l'évaluation
Resultat: dataframe, index: etudid ayant une note, valeur: note brute.
"""
eval_df = pd.read_sql_query(
"""SELECT n.etudid, n.value AS "%(evaluation_id)s"
FROM notes_notes n, notes_moduleimpl_inscription i
WHERE evaluation_id=%(evaluation_id)s
AND n.etudid = i.etudid
AND i.moduleimpl_id = %(moduleimpl_id)s
""",
db.engine,
params={
"evaluation_id": evaluation.id,
"moduleimpl_id": evaluation.moduleimpl.id,
},
index_col="etudid",
)
with db.engine.begin() as connection:
eval_df = pd.read_sql_query(
self._load_evaluation_notes_q,
connection,
params={
"evaluation_id": evaluation.id,
"moduleimpl_id": evaluation.moduleimpl.id,
},
index_col="etudid",
)
eval_df[str(evaluation.id)] = pd.to_numeric(eval_df[str(evaluation.id)])
return eval_df

22
app/models/ues.py
View File

@ -206,20 +206,14 @@ class UniteEns(db.Model):
Si niveau est None, désassocie.
"""
if niveau is not None:
self._check_apc_conflict(niveau.id, self.parcour_id)
# Le niveau est-il dans le parcours ? Sinon, erreur
if self.parcour and niveau.id not in (
n.id
for n in niveau.niveaux_annee_de_parcours(
self.parcour, self.annee(), self.formation.referentiel_competence
)
):
log(
f"set_niveau_competence: niveau {niveau} hors parcours {self.parcour}"
)
return
if niveau.id == self.niveau_competence_id:
return True # nothing to do
if (niveau is not None) and (self.niveau_competence_id is not None):
ok, error_message = self.check_niveau_unique_dans_parcours(
niveau, self.parcours
)
if not ok:
return ok, error_message
self.niveau_competence = niveau
db.session.add(self)

5
app/scodoc/gen_tables.py
View File

@ -40,7 +40,6 @@ Par exemple, la clé '_css_row_class' spécifie le style CSS de la ligne.
"""
from __future__ import print_function
import random
from collections import OrderedDict
from xml.etree import ElementTree
@ -60,7 +59,7 @@ from app.scodoc import sco_pdf
from app.scodoc import sco_xml
from app.scodoc.sco_exceptions import ScoPDFFormatError
from app.scodoc.sco_pdf import SU
from app import log
from app import log, ScoDocJSONEncoder
def mark_paras(L, tags) -> list[str]:
@ -647,7 +646,7 @@ class GenTable(object):
# v = str(v)
r[cid] = v
d.append(r)
return json.dumps(d, cls=scu.ScoDocJSONEncoder)
return json.dumps(d, cls=ScoDocJSONEncoder)
def make_page(
self,

4
app/scodoc/sco_archives.py
View File

@ -64,7 +64,7 @@ from flask import flash, g, request, url_for
import app.scodoc.sco_utils as scu
from config import Config
from app import log
from app import log, ScoDocJSONEncoder
from app.but import jury_but_pv
from app.comp import res_sem
from app.comp.res_compat import NotesTableCompat
@ -365,7 +365,7 @@ def do_formsemestre_archive(
# Bulletins en JSON
data = gen_formsemestre_recapcomplet_json(formsemestre_id, xml_with_decisions=True)
data_js = json.dumps(data, indent=1, cls=scu.ScoDocJSONEncoder)
data_js = json.dumps(data, indent=1, cls=ScoDocJSONEncoder)
if data:
PVArchive.store(archive_id, "Bulletins.json", data_js)
# Décisions de jury, en XLS

3
app/scodoc/sco_bulletins_json.py
View File

@ -33,6 +33,7 @@ import json
from flask import abort
from app import ScoDocJSONEncoder
from app.comp import res_sem
from app.comp.res_compat import NotesTableCompat
from app.models import but_validations
@ -74,7 +75,7 @@ def make_json_formsemestre_bulletinetud(
version=version,
)
return json.dumps(d, cls=scu.ScoDocJSONEncoder)
return json.dumps(d, cls=ScoDocJSONEncoder)
# (fonction séparée: n'utilise pas formsemestre_bulletinetud_dict()

5
app/scodoc/sco_preferences.py
View File

@ -111,7 +111,7 @@ get_base_preferences(formsemestre_id)
"""
import flask
from flask import g, request, url_for
from flask import current_app, g, request, url_for
# from flask_login import current_user
@ -2000,7 +2000,8 @@ class BasePreferences(object):
value = _get_pref_default_value_from_config(name, pref[1])
self.default[name] = value
self.prefs[None][name] = value
log(f"creating missing preference for {name}={value}")
if not current_app.testing:
log(f"creating missing preference for {name}={value}")
# add to db table
self._editor.create(
cnx, {"dept_id": self.dept_id, "name": name, "value": value}

7
app/scodoc/sco_saisie_notes.py
View File

@ -152,7 +152,7 @@ def _check_notes(notes: list[(int, float)], evaluation: dict, mod: dict):
absents = [] # etudid absents
tosuppress = [] # etudids avec ancienne note à supprimer
for (etudid, note) in notes:
for etudid, note in notes:
note = str(note).strip().upper()
try:
etudid = int(etudid) #
@ -536,7 +536,7 @@ def notes_add(
evaluation_id, getallstudents=True, include_demdef=True
)
}
for (etudid, value) in notes:
for etudid, value in notes:
if check_inscription and (etudid not in inscrits):
raise NoteProcessError(f"etudiant {etudid} non inscrit dans ce module")
if (value is not None) and not isinstance(value, float):
@ -556,7 +556,7 @@ def notes_add(
[]
) # etudids pour lesquels il y a une decision de jury et que la note change
try:
for (etudid, value) in notes:
for etudid, value in notes:
changed = False
if etudid not in notes_db:
# nouvelle note
@ -657,6 +657,7 @@ def notes_add(
formsemestre_id=M["formsemestre_id"]
) # > modif notes (exception)
sco_cache.EvaluationCache.delete(evaluation_id)
raise # XXX
raise ScoGenError("Erreur enregistrement note: merci de ré-essayer") from exc
if do_it:
cnx.commit()

14
app/scodoc/sco_utils.py
View File

@ -61,8 +61,8 @@ from flask import flash, url_for, make_response, jsonify
from werkzeug.http import HTTP_STATUS_CODES
from config import Config
from app import log
from app.scodoc.sco_vdi import ApoEtapeVDI
from app import log, ScoDocJSONEncoder
from app.scodoc.codes_cursus import NOTES_TOLERANCE, CODES_EXPL
from app.scodoc import sco_xml
import sco_version
@ -855,16 +855,6 @@ def sendPDFFile(data, filename): # DEPRECATED utiliser send_file
return send_file(data, filename=filename, mime=PDF_MIMETYPE, attached=True)
class ScoDocJSONEncoder(flask.json.provider.DefaultJSONProvider):
def default(self, o): # pylint: disable=E0202
if isinstance(o, (datetime.date, datetime.datetime)):
return o.isoformat()
elif isinstance(o, ApoEtapeVDI):
return str(o)
else:
return json.JSONEncoder.default(self, o)
def sendJSON(data, attached=False, filename=None):
js = json.dumps(data, indent=1, cls=ScoDocJSONEncoder)
return send_file(

51
migrations/env.py
View File

@ -1,9 +1,10 @@
from __future__ import with_statement
# Copied 2023-04-03 from
# https://raw.githubusercontent.com/miguelgrinberg/Flask-Migrate/main/src/flask_migrate/templates/flask/env.py
import logging
from logging.config import fileConfig
from flask import current_app
import flask_sqlalchemy
from alembic import context
@ -14,17 +15,31 @@ config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
logger = logging.getLogger('alembic.env')
logger = logging.getLogger("alembic.env")
def get_engine():
if int(flask_sqlalchemy.__version__[0]) < 3: # <--------- MODIFIED By EMMANUEL
# this works with Flask-SQLAlchemy<3 and Alchemical
return current_app.extensions["migrate"].db.get_engine()
else:
# this works with Flask-SQLAlchemy>=3
return current_app.extensions["migrate"].db.engine
def get_engine_url():
try:
return get_engine().url.render_as_string(hide_password=False).replace("%", "%%")
except AttributeError:
return str(get_engine().url).replace("%", "%%")
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
config.set_main_option(
'sqlalchemy.url',
str(current_app.extensions['migrate'].db.get_engine().url).replace(
'%', '%%'))
target_metadata = current_app.extensions['migrate'].db.metadata
config.set_main_option("sqlalchemy.url", get_engine_url())
target_db = current_app.extensions["migrate"].db
# other values from the config, defined by the needs of env.py,
# can be acquired:
@ -32,6 +47,12 @@ target_metadata = current_app.extensions['migrate'].db.metadata
# ... etc.
def get_metadata():
if hasattr(target_db, "metadatas"):
return target_db.metadatas[None]
return target_db.metadata
def run_migrations_offline():
"""Run migrations in 'offline' mode.
@ -45,9 +66,7 @@ def run_migrations_offline():
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url, target_metadata=target_metadata, literal_binds=True
)
context.configure(url=url, target_metadata=get_metadata(), literal_binds=True)
with context.begin_transaction():
context.run_migrations()
@ -65,20 +84,20 @@ def run_migrations_online():
# when there are no changes to the schema
# reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
if getattr(config.cmd_opts, "autogenerate", False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
logger.info("No changes in schema detected.")
connectable = current_app.extensions['migrate'].db.get_engine()
connectable = get_engine()
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
target_metadata=get_metadata(),
process_revision_directives=process_revision_directives,
**current_app.extensions['migrate'].configure_args
**current_app.extensions["migrate"].configure_args
)
with context.begin_transaction():

6
tests/unit/sco_fake_gen.py
View File

@ -178,7 +178,7 @@ class ScoFake(object):
self,
formation_id=None,
acronyme=None,
numero=None,
numero=0,
titre="",
type=None,
ue_code=None,
@ -200,7 +200,7 @@ class ScoFake(object):
return oid
@logging_meth
def create_matiere(self, ue_id=None, titre=None, numero=None) -> int:
def create_matiere(self, ue_id=None, titre=None, numero=0) -> int:
oid = sco_edit_matiere.do_matiere_create(locals())
oids = sco_edit_matiere.matiere_list(args={"matiere_id": oid})
if not oids:
@ -218,7 +218,7 @@ class ScoFake(object):
coefficient=None,
matiere_id=None,
semestre_id=1,
numero=None,
numero=0,
abbrev=None,
ects=None,
code_apogee=None,