dubacq/ScoDoc
1
0
Fork 0
forked from ScoDoc/ScoDoc
Code Pull Requests Activity

Génération bulletin BUT json

Browse Source
This commit is contained in:
Emmanuel Viennet 2021-12-05 20:21:51 +01:00
parent 1a673862aa
commit 3ba30f6250
12 changed files with 328 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

224
app/but/bulletin_but.py Normal file
View File

@ -0,0 +1,224 @@
##############################################################################
# ScoDoc
# Copyright (c) 1999 - 2021 Emmanuel Viennet. All rights reserved.
# See LICENSE
##############################################################################
import datetime
import numpy as np
import pandas as pd
from app import db
from app.comp import df_cache, moy_ue, moy_mod, inscr_mod
from app.scodoc import sco_utils as scu
from app.scodoc.sco_cache import ResultatsSemestreBUTCache
from app.scodoc.sco_exceptions import ScoFormatError
from app.scodoc.sco_utils import jsnan
class ResultatsSemestreBUT:
"""Structure légère pour stocker les résultats du semestre et
générer les bulletins.
__init__ : charge depuis le cache ou calcule
invalidate(): invalide données cachées
"""
_cached_attrs = (
"sem_cube",
"modimpl_inscr_df",
"modimpl_coefs_df",
"etud_moy_ue",
"modimpls_evals_poids",
"modimpls_evals_notes",
)
def __init__(self, formsemestre):
self.formsemestre = formsemestre
self.ues = formsemestre.query_ues().all()
self.modimpls = formsemestre.modimpls.all()
self.etuds = self.formsemestre.etuds.all()
self.etud_index = {e.id: idx for idx, e in enumerate(self.etuds)}
self.saes = [
m for m in self.modimpls if m.module.module_type == scu.ModuleType.SAE
]
self.ressources = [
m for m in self.modimpls if m.module.module_type == scu.ModuleType.RESSOURCE
]
if not self.load_cached():
self.compute()
self.store()
def load_cached(self) -> bool:
"Load cached dataframes, returns False si pas en cache"
data = ResultatsSemestreBUTCache.get(self.formsemestre.id)
if not data:
return False
for attr in self._cached_attrs:
setattr(self, attr, data[attr])
return True
def store(self):
"Cache our dataframes"
ResultatsSemestreBUTCache.set(
self.formsemestre.id,
{attr: getattr(self, attr) for attr in self._cached_attrs},
)
def compute(self):
"Charge les notes et inscriptions et calcule toutes les moyennes"
(
self.sem_cube,
self.modimpls_evals_poids,
self.modimpls_evals_notes,
_,
) = moy_ue.notes_sem_load_cube(self.formsemestre)
self.modimpl_inscr_df = inscr_mod.df_load_modimpl_inscr(self.formsemestre)
self.modimpl_coefs_df, _, _ = moy_ue.df_load_modimpl_coefs(
self.formsemestre, ues=self.ues, modimpls=self.modimpls
)
# l'idx de la colonne du mod modimpl.id est
# modimpl_coefs_df.columns.get_loc(modimpl.id)
# idx de l'UE: modimpl_coefs_df.index.get_loc(ue.id)
self.etud_moy_ue = moy_ue.compute_ue_moys(
self.sem_cube,
self.etuds,
self.modimpls,
self.ues,
self.modimpl_inscr_df,
self.modimpl_coefs_df,
)
def etud_ue_mod_results(self, etud, ue, modimpls) -> dict:
"dict synthèse résultats dans l'UE pour les modules indiqués"
d = {}
etud_idx = self.etud_index[etud.id]
ue_idx = self.modimpl_coefs_df.index.get_loc(ue.id)
etud_moy_module = self.sem_cube[etud_idx] # module x UE
for mi in modimpls:
d[mi.module.code] = {
"id": mi.id,
"coef": self.modimpl_coefs_df[mi.id][ue.id],
"moyenne": jsnan(
etud_moy_module[self.modimpl_coefs_df.columns.get_loc(mi.id)][
ue_idx
]
),
}
return d
def etud_ue_results(self, etud, ue):
"dict synthèse résultats UE"
d = {
"id": ue.id,
"ECTS": {
"acquis": 0, # XXX TODO voir jury
"total": ue.ects,
},
"competence": None, # XXX TODO lien avec référentiel
"moyenne": jsnan(self.etud_moy_ue[ue.id].mean()),
"bonus": None, # XXX TODO
"malus": None, # XXX TODO voir ce qui est ici
"capitalise": None, # "AAAA-MM-JJ" TODO
"ressources": self.etud_ue_mod_results(etud, ue, self.ressources),
"saes": self.etud_ue_mod_results(etud, ue, self.saes),
}
return d
def etud_mods_results(self, etud, modimpls) -> dict:
"""dict synthèse résultats des modules indiqués,
avec évaluations de chacun."""
d = {}
etud_idx = self.etud_index[etud.id]
for mi in modimpls:
mod_idx = self.modimpl_coefs_df.columns.get_loc(mi.id)
# moyennes indicatives (moyennes de moyennes d'UE)
moyennes_etuds = np.nan_to_num(
self.sem_cube[:, mod_idx, :].mean(axis=1),
copy=False,
)
d[mi.module.code] = {
"id": mi.id,
"titre": mi.module.titre,
"code_apogee": mi.module.code_apogee,
"moyenne": {
"value": jsnan(self.sem_cube[etud_idx, mod_idx].mean()),
"min": jsnan(moyennes_etuds.min()),
"max": jsnan(moyennes_etuds.max()),
"moy": jsnan(moyennes_etuds.mean()),
},
"evaluations": [
self.etud_eval_results(etud, e)
for e in mi.evaluations
if e.visibulletin
],
}
return d
def etud_eval_results(self, etud, e) -> dict:
"dict resultats d'un étudiant à une évaluation"
eval_notes = self.modimpls_evals_notes[e.moduleimpl_id][str(e.id)] # pd.Series
notes_ok = eval_notes.where(eval_notes > -1000).dropna()
d = {
"id": e.id,
"description": e.description,
"date": e.jour.isoformat(),
"heure_debut": e.heure_debut.strftime("%H:%M") if e.heure_debut else None,
"heure_fin": e.heure_fin.strftime("%H:%M") if e.heure_debut else None,
"coef": e.coefficient,
"poids": {p.ue.acronyme: p.poids for p in e.ue_poids},
"note": {
"value": jsnan(
self.modimpls_evals_notes[e.moduleimpl_id][str(e.id)][etud.id]
),
"min": jsnan(notes_ok.min()),
"max": jsnan(notes_ok.max()),
"moy": jsnan(notes_ok.mean()),
},
}
return d
def bulletin_etud(self, etud, formsemestre) -> dict:
"""Le bulletin de l'étudiant dans ce semestre"""
d = {
"version": "0",
"type": "BUT",
"date": datetime.datetime.utcnow().isoformat() + "Z",
"etudiant": etud.to_dict_bul(),
"formation": {
"id": formsemestre.formation.id,
"acronyme": formsemestre.formation.acronyme,
"titre_officiel": formsemestre.formation.titre_officiel,
"titre": formsemestre.formation.titre,
},
"formsemestre_id": formsemestre.id,
"ressources": None, # XXX TODO
"saes": None, # XXX TODO
"ues": {ue.acronyme: self.etud_ue_results(etud, ue) for ue in self.ues},
"semestre": {
"notes": { # moyenne des moyennes générales du semestre
"value": jsnan("xxx"), # XXX TODO
"min": jsnan("0."),
"moy": jsnan("10.0"),
"max": jsnan("20.00"),
},
"rang": { # classement wrt moyenne général, indicatif
"value": None, # XXX TODO
"total": None,
},
"absences": { # XXX TODO
"injustifie": 1,
"total": 33,
},
"date_debut": formsemestre.date_debut.isoformat(),
"date_fin": formsemestre.date_fin.isoformat(),
"annee_universitaire": self.formsemestre.annee_scolaire_str(),
"inscription": "TODO-MM-JJ", # XXX TODO
"numero": formsemestre.semestre_id,
"decision": None, # XXX TODO
"situation": "Décision jury: Validé. Diplôme obtenu.", # XXX TODO
"date_jury": "AAAA-MM-JJ", # XXX TODO
"groupes": [], # XXX TODO
},
}
return d

25
app/comp/inscr_mod.py
View File

@ -14,16 +14,15 @@ from app import models
# sur test debug 116 etuds, 18 modules, on est autour de 250ms.
# On a testé trois approches, ci-dessous (et retenu la 1ere)
#
def df_load_modimpl_inscr(formsemestre_id: int) -> pd.DataFrame:
def df_load_modimpl_inscr(formsemestre) -> pd.DataFrame:
"""Charge la matrice des inscriptions aux modules du semestre
rows: etudid
columns: moduleimpl_id (en chaîne)
value: bool (0/1 inscrit ou pas)
"""
# méthode la moins lente: une requete par module, merge les dataframes
sem = models.FormSemestre.query.get(formsemestre_id)
moduleimpl_ids = [m.id for m in sem.modimpls]
etudids = [i.etudid for i in sem.inscriptions]
moduleimpl_ids = [m.id for m in formsemestre.modimpls]
etudids = [i.etudid for i in formsemestre.inscriptions]
df = pd.DataFrame(index=etudids, dtype=int)
for moduleimpl_id in moduleimpl_ids:
ins_df = pd.read_sql_query(
@ -46,27 +45,25 @@ def df_load_modimpl_inscr(formsemestre_id: int) -> pd.DataFrame:
# timeit.timeit('x = df_load_module_inscr_v0(696)', number=100, globals=globals())
def df_load_modimpl_inscr_v0(formsemestre_id: int):
def df_load_modimpl_inscr_v0(formsemestre):
# methode 0, pur SQL Alchemy, 1.5 à 2 fois plus lente
sem = models.FormSemestre.query.get(formsemestre_id)
moduleimpl_ids = [m.id for m in sem.modimpls]
etudids = [i.etudid for i in sem.inscriptions]
moduleimpl_ids = [m.id for m in formsemestre.modimpls]
etudids = [i.etudid for i in formsemestre.inscriptions]
df = pd.DataFrame(False, columns=moduleimpl_ids, index=etudids, dtype=bool)
for modimpl in sem.modimpls:
for modimpl in formsemestre.modimpls:
ins_mod = df[modimpl.id]
for inscr in modimpl.inscriptions:
ins_mod[inscr.etudid] = True
return df # x100 30.7s 46s 32s
def df_load_modimpl_inscr_v2(formsemestre_id):
sem = models.FormSemestre.query.get(formsemestre_id)
moduleimpl_ids = [m.id for m in sem.modimpls]
etudids = [i.etudid for i in sem.inscriptions]
def df_load_modimpl_inscr_v2(formsemestre):
moduleimpl_ids = [m.id for m in formsemestre.modimpls]
etudids = [i.etudid for i in formsemestre.inscriptions]
df = pd.DataFrame(False, columns=moduleimpl_ids, index=etudids, dtype=bool)
cursor = db.engine.execute(
"select moduleimpl_id, etudid from notes_moduleimpl_inscription i, notes_moduleimpl m where i.moduleimpl_id = m.id and m.formsemestre_id = %(formsemestre_id)s",
{"formsemestre_id": formsemestre_id},
{"formsemestre_id": formsemestre.id},
)
for moduleimpl_id, etudid in cursor:
df[moduleimpl_id][etudid] = True

28
app/comp/moy_ue.py
View File

@ -75,18 +75,22 @@ def df_load_module_coefs(formation_id: int, semestre_idx: int = None) -> pd.Data
return module_coefs_df, ues, modules
def df_load_modimpl_coefs(formsemestre: models.FormSemestre) -> pd.DataFrame:
def df_load_modimpl_coefs(
formsemestre: models.FormSemestre, ues=None, modimpls=None
) -> pd.DataFrame:
"""Charge les coefs des modules du formsemestre indiqué.
Comme df_load_module_coefs mais prend seulement les UE
et modules du formsemestre.
Si ues et modimpls sont None, prend tous ceux du formsemestre.
Résultat: (module_coefs_df, ues, modules)
DataFrame rows = UEs, columns = modimpl, value = coef.
"""
ues = formsemestre.query_ues().all()
if ues is None:
ues = formsemestre.query_ues().all()
ue_ids = [x.id for x in ues]
modimpls = formsemestre.modimpls.all()
if modimpls is None:
modimpls = formsemestre.modimpls.all()
modimpl_ids = [x.id for x in modimpls]
mod2impl = {m.module.id: m.id for m in modimpls}
modimpl_coefs_df = pd.DataFrame(columns=modimpl_ids, index=ue_ids, dtype=float)
@ -115,13 +119,15 @@ def notes_sem_assemble_cube(modimpls_notes: list[pd.DataFrame]) -> np.ndarray:
return modimpls_notes.swapaxes(0, 1)
def notes_sem_load_cube(formsemestre_id):
def notes_sem_load_cube(formsemestre):
"""Calcule le cube des notes du semestre
(charge toutes les notes, calcule les moyenne des modules
et assemble le cube)
Resultat: ndarray (etuds x modimpls x UEs)
"""
formsemestre = FormSemestre.query.get(formsemestre_id)
modimpls_evals_poids = {} # modimpl.id : evals_poids
modimpls_evals_notes = {} # modimpl.id : evals_notes
modimpls_evaluations = {} # modimpl.id : liste des évaluations
modimpls_notes = []
for modimpl in formsemestre.modimpls:
evals_notes, evaluations = moy_mod.df_load_modimpl_notes(modimpl.id)
@ -129,8 +135,16 @@ def notes_sem_load_cube(formsemestre_id):
etuds_moy_module = moy_mod.compute_module_moy(
evals_notes, evals_poids, evaluations
)
modimpls_evals_poids[modimpl.id] = evals_poids
modimpls_evals_notes[modimpl.id] = evals_notes
modimpls_evaluations[modimpl.id] = evaluations
modimpls_notes.append(etuds_moy_module)
return notes_sem_assemble_cube(modimpls_notes)
return (
notes_sem_assemble_cube(modimpls_notes),
modimpls_evals_poids,
modimpls_evals_notes,
modimpls_evaluations,
)
def compute_ue_moys(

23
app/models/etudiants.py
View File

@ -8,6 +8,7 @@ from app import db
from app.models import APO_CODE_STR_LEN
from app.models import SHORT_STR_LEN
from app.models import CODE_STR_LEN
from app.scodoc import sco_photos
class Identite(db.Model):
@ -44,6 +45,7 @@ class Identite(db.Model):
# ne pas utiliser après migrate_scodoc7_dept_archives
scodoc7_id = db.Column(db.Text(), nullable=True)
#
adresses = db.relationship("Adresse", lazy="dynamic", backref="etud")
billets = db.relationship("BilletAbsence", backref="etudiant", lazy="dynamic")
def __repr__(self):
@ -64,6 +66,27 @@ class Identite(db.Model):
else:
return self.nom
def to_dict_bul(self):
"""Infos exportées dans les bulletins"""
return {
"civilite": self.civilite,
"code_ine": self.code_nip,
"code_nip": self.code_ine,
"date_naissance": self.date_naissance.isoformat()
if self.date_naissance
else None,
"email": self.adresses[0].email or None
if self.adresses.count() > 0
else None,
"emailperso": self.adresses[0].emailperso or None
if self.adresses.count() > 0
else None,
"etudid": self.id,
"nom": self.nom_disp(),
"photo_url": sco_photos.get_etud_photo_url(self.id),
"prenom": self.prenom,
}
def inscription_courante(self):
"""La première inscription à un formsemestre _actuellement_ en cours.
None s'il n'y en a pas (ou plus, ou pas encore).

2
app/models/evaluations.py
View File

@ -46,7 +46,7 @@ class Evaluation(db.Model):
ues = db.relationship("UniteEns", secondary="evaluation_ue_poids", viewonly=True)
def __repr__(self):
return f"<Evaluation {self.id} {self.jour.isoformat() if self.jour else ''}{self.description[:16] if self.description else ''}"
return f"""<Evaluation {self.id} {self.jour.isoformat() if self.jour else ''} "{self.description[:16] if self.description else ''}">"""
def to_dict(self):
e = dict(self.__dict__)

4
app/models/formations.py
View File

@ -58,6 +58,10 @@ class Formation(db.Model):
"""get l'instance de TypeParcours de cette formation"""
return sco_codes_parcours.get_parcours_from_code(self.type_parcours)
def is_apc(self):
"True si formation APC avec SAE (BUT)"
return self.get_parcours().APC_SAE
def get_module_coefs(self, semestre_idx: int = None):
"""Les coefs des modules vers les UE (accès via cache)"""
from app.comp import moy_ue

4
app/models/formsemestre.py
View File

@ -178,6 +178,10 @@ class FormSemestre(db.Model):
else:
return ", ".join([u.get_nomcomplet() for u in self.responsables])
def annee_scolaire_str(self):
"2021 - 2022"
return scu.annee_scolaire_repr(self.date_debut.year, self.date_debut.month)
def session_id(self) -> str:
"""identifiant externe de semestre de formation
Exemple: RT-DUT-FI-S1-ANNEE

11
app/scodoc/sco_cache.py
View File

@ -155,6 +155,16 @@ class EvaluationCache(ScoDocCache):
cls.delete_many(evaluation_ids)
class ResultatsSemestreBUTCache(ScoDocCache):
"""Cache pour les résultats ResultatsSemestreBUT.
Clé: formsemestre_id
Valeur: { un paquet de dataframes }
"""
prefix = "RBUT"
timeout = 1 * 60 # ttl 1 minutes (en phase de mise au point)
class AbsSemEtudCache(ScoDocCache):
"""Cache pour les comptes d'absences d'un étudiant dans un semestre.
Ce cache étant indépendant des semestres, le compte peut être faux lorsqu'on
@ -289,6 +299,7 @@ def invalidate_formsemestre( # was inval_cache(formsemestre_id=None, pdfonly=Fa
SemInscriptionsCache.delete_many(formsemestre_ids)
SemBulletinsPDFCache.invalidate_sems(formsemestre_ids)
ResultatsSemestreBUTCache.delete_many(formsemestre_ids)
class DefferedSemCacheManager:

13
app/scodoc/sco_photos.py
View File

@ -43,7 +43,7 @@ Les images sont servies par ScoDoc, via la méthode getphotofile?etudid=xxx
"""
from flask.helpers import make_response
from flask.helpers import make_response, url_for
from app.scodoc.sco_exceptions import ScoGenError
import datetime
import glob
@ -91,14 +91,17 @@ def photo_portal_url(etud):
return None
def get_etud_photo_url(etudid, size="small"):
return url_for(
"scolar.get_photo_image", scodoc_dept=g.scodoc_dept, etudid=etudid, size=size
)
def etud_photo_url(etud, size="small", fast=False):
"""url to the image of the student, in "small" size or "orig" size.
If ScoDoc doesn't have an image and a portal is configured, link to it.
"""
photo_url = scu.ScoURL() + "/get_photo_image?etudid=%s&size=%s" % (
etud["etudid"],
size,
)
photo_url = get_etud_photo_url(etud["etudid"], size=size)
if fast:
return photo_url
path = photo_pathname(etud, size=size)

7
app/scodoc/sco_utils.py
View File

@ -202,6 +202,13 @@ def isnumber(x):
return isinstance(x, numbers.Number)
def jsnan(x):
"if x is NaN, returns None"
if isinstance(x, numbers.Number) and np.isnan(x):
return None
return x
def join_words(*words):
words = [str(w).strip() for w in words if w is not None]
return " ".join([w for w in words if w])

12
app/views/notes.py
View File

@ -38,10 +38,11 @@ from operator import itemgetter
from xml.etree import ElementTree
import flask
from flask import url_for
from flask import url_for, jsonify
from flask import current_app, g, request
from flask_login import current_user
from werkzeug.utils import redirect
from app.models.formsemestre import FormSemestre
from config import Config
@ -49,7 +50,7 @@ from app import api
from app import db
from app import models
from app.auth.models import User
from app.but import bulletin_but
from app.decorators import (
scodoc,
scodoc7func,
@ -285,6 +286,13 @@ def formsemestre_bulletinetud(
raise ScoValueError("Paramètre manquant: spécifier code_nip ou etudid")
if not formsemestre_id:
raise ScoValueError("Paramètre manquant: formsemestre_id est requis")
formsemestre = FormSemestre.query.get_or_404(formsemestre_id)
if formsemestre.formation.is_apc():
etud = models.Identite.query.get_or_404(etudid)
r = bulletin_but.ResultatsSemestreBUT(formsemestre)
return jsonify(r.bulletin_etud(etud, formsemestre))
return sco_bulletins.formsemestre_bulletinetud(
etudid=etudid,
formsemestre_id=formsemestre_id,

8
tests/unit/test_but_ues.py
View File

@ -53,7 +53,7 @@ def test_ue_moy(test_client):
# Les moduleimpls
modimpls = [evaluation1.moduleimpl, evaluation2.moduleimpl]
# Check inscriptions modules
modimpl_inscr_df = inscr_mod.df_load_modimpl_inscr(formsemestre_id)
modimpl_inscr_df = inscr_mod.df_load_modimpl_inscr(formsemestre)
assert (modimpl_inscr_df.values == np.array([[1, 1]])).all()
# Coefs des modules vers les UE:
modimpl_coefs_df, ues, modimpls = moy_ue.df_load_modimpl_coefs(formsemestre)
@ -69,7 +69,7 @@ def test_ue_moy(test_client):
_ = sco_saisie_notes.notes_add(G.default_user, evaluation1.id, [(etudid, n1)])
_ = sco_saisie_notes.notes_add(G.default_user, evaluation2.id, [(etudid, n2)])
# Recalcul des moyennes
sem_cube = moy_ue.notes_sem_load_cube(formsemestre_id)
sem_cube, _, _, _ = moy_ue.notes_sem_load_cube(formsemestre)
etuds = formsemestre.etuds.all()
etud_moy_ue = moy_ue.compute_ue_moys(
sem_cube, etuds, modimpls, ues, modimpl_inscr_df, modimpl_coefs_df
@ -103,7 +103,7 @@ def test_ue_moy(test_client):
).first()
db.session.delete(inscr)
db.session.commit()
modimpl_inscr_df = inscr_mod.df_load_modimpl_inscr(formsemestre_id)
modimpl_inscr_df = inscr_mod.df_load_modimpl_inscr(formsemestre)
assert (modimpl_inscr_df.values == np.array([[1, 0]])).all()
n1, n2 = 5.0, NOTES_NEUTRALISE
# On ne doit pas pouvoir saisir de note sans être inscrit:
@ -114,7 +114,7 @@ def test_ue_moy(test_client):
exception_raised = True
assert exception_raised
# Recalcule les notes:
sem_cube = moy_ue.notes_sem_load_cube(formsemestre_id)
sem_cube, _, _, _ = moy_ue.notes_sem_load_cube(formsemestre)
etuds = formsemestre.etuds.all()
etud_moy_ue = moy_ue.compute_ue_moys(
sem_cube, etuds, modimpls, ues, modimpl_inscr_df, modimpl_coefs_df