forked from ScoDoc/ScoDoc
452 lines
14 KiB
Python
452 lines
14 KiB
Python
# -*- mode: python -*-
|
|
# -*- coding: utf-8 -*-
|
|
|
|
##############################################################################
|
|
#
|
|
# Gestion scolarite IUT
|
|
#
|
|
# Copyright (c) 1999 - 2024 Emmanuel Viennet. All rights reserved.
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation; either version 2 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
#
|
|
# Emmanuel Viennet emmanuel.viennet@viennet.net
|
|
#
|
|
##############################################################################
|
|
|
|
"""Gestion des caches
|
|
|
|
Ré-écrite pour ScoDoc8, utilise flask_caching et REDIS
|
|
|
|
ScoDoc est maintenant multiprocessus / mono-thread, avec un cache partagé.
|
|
"""
|
|
|
|
|
|
# API pour les caches:
|
|
# sco_cache.MyCache.get( formsemestre_id)
|
|
# => sco_cache.MyCache.get(formsemestre_id)
|
|
#
|
|
# sco_cache.MyCache.delete(formsemestre_id)
|
|
# sco_cache.MyCache.delete_many(formsemestre_id_list)
|
|
#
|
|
# Bulletins PDF:
|
|
# sco_cache.SemBulletinsPDFCache.get(formsemestre_id, version)
|
|
# sco_cache.SemBulletinsPDFCache.set(formsemestre_id, version, filename, pdfdoc)
|
|
# sco_cache.SemBulletinsPDFCache.delete(formsemestre_id) suppr. toutes les versions
|
|
|
|
# Evaluations:
|
|
# sco_cache.EvaluationCache.get(evaluation_id), set(evaluation_id, value), delete(evaluation_id),
|
|
#
|
|
|
|
import traceback
|
|
|
|
from flask import g
|
|
|
|
import app
|
|
from app import db, log
|
|
from app.scodoc import notesdb as ndb
|
|
from app.scodoc import sco_utils as scu
|
|
from app.scodoc.sco_exceptions import ScoException
|
|
|
|
CACHE = None # set in app.__init__.py
|
|
|
|
|
|
class ScoDocCache:
|
|
"""Cache for ScoDoc objects.
|
|
keys are prefixed by the current departement: g.scodoc_dept MUST be set.
|
|
"""
|
|
|
|
timeout = None # ttl, infinite by default
|
|
prefix = ""
|
|
verbose = False # if true, verbose logging (debug)
|
|
|
|
@classmethod
|
|
def _get_key(cls, oid):
|
|
return g.scodoc_dept + "_" + cls.prefix + "_" + str(oid)
|
|
|
|
@classmethod
|
|
def get(cls, oid):
|
|
"""Returns cached object, or None"""
|
|
key = cls._get_key(oid)
|
|
try:
|
|
return CACHE.get(key)
|
|
except:
|
|
log(f"XXX CACHE Warning: error in get(key={key})")
|
|
log(traceback.format_exc())
|
|
return None
|
|
|
|
@classmethod
|
|
def set(cls, oid, value):
|
|
"""Store value"""
|
|
key = cls._get_key(oid)
|
|
if cls.verbose:
|
|
log(
|
|
f"{cls.__name__}.set key={key}, type={type(value).__name__}, timeout={cls.timeout}"
|
|
)
|
|
try:
|
|
status = CACHE.set(key, value, timeout=cls.timeout)
|
|
if not status:
|
|
log("Error: cache set failed !")
|
|
except Exception as exc:
|
|
log("XXX CACHE Warning: error in set !!!")
|
|
log(exc)
|
|
status = None
|
|
return status
|
|
|
|
@classmethod
|
|
def delete(cls, oid):
|
|
"""Remove from cache"""
|
|
# if cls.verbose:
|
|
# log(f"{cls.__name__}.delete({oid})")
|
|
CACHE.delete(cls._get_key(oid))
|
|
|
|
@classmethod
|
|
def delete_many(cls, oids):
|
|
"""Remove multiple keys at once"""
|
|
if cls.verbose:
|
|
log(f"{cls.__name__}.delete_many({oids})")
|
|
# delete_many seems bugged:
|
|
# CACHE.delete_many([cls._get_key(oid) for oid in oids])
|
|
for oid in oids:
|
|
cls.delete(oid)
|
|
|
|
@classmethod
|
|
def delete_pattern(cls, pattern: str, std_prefix=True) -> int:
|
|
"""Delete all keys matching pattern.
|
|
The pattern starts with flask_cache_<dept_acronym>.
|
|
If std_prefix is true (default), the prefix is added
|
|
to the given pattern.
|
|
Examples:
|
|
'TABASSI_tableau-etud-1234:*'
|
|
Or, with std_prefix false, 'flask_cache_RT_TABASSI_tableau-etud-1234:*'
|
|
|
|
Returns number of keys deleted.
|
|
"""
|
|
# see https://stackoverflow.com/questions/36708461/flask-cache-list-keys-based-on-a-pattern
|
|
assert CACHE.cache.__class__.__name__ == "RedisCache" # Redis specific
|
|
import redis
|
|
|
|
if std_prefix:
|
|
pattern = "flask_cache_" + g.scodoc_dept + "_" + cls.prefix + "_" + pattern
|
|
|
|
r = redis.Redis()
|
|
count = 0
|
|
for key in r.scan_iter(pattern):
|
|
log(f"{cls.__name__}.delete_pattern({key})")
|
|
r.delete(key)
|
|
count += 1
|
|
return count
|
|
|
|
|
|
class EvaluationCache(ScoDocCache):
|
|
"""Cache for evaluations.
|
|
Clé: evaluation_id
|
|
Valeur: { 'etudid' : note }
|
|
"""
|
|
|
|
prefix = "EVAL"
|
|
|
|
@classmethod
|
|
def invalidate_sem(cls, formsemestre_id):
|
|
"delete evaluations in this formsemestre from cache"
|
|
from app.models.evaluations import Evaluation
|
|
from app.models.moduleimpls import ModuleImpl
|
|
|
|
evaluation_ids = [
|
|
e.id
|
|
for e in Evaluation.query.join(ModuleImpl).filter_by(
|
|
formsemestre_id=formsemestre_id
|
|
)
|
|
]
|
|
cls.delete_many(evaluation_ids)
|
|
|
|
@classmethod
|
|
def invalidate_all_sems(cls):
|
|
"delete all evaluations in current dept from cache"
|
|
evaluation_ids = [
|
|
x[0]
|
|
for x in ndb.SimpleQuery(
|
|
"""SELECT e.id
|
|
FROM notes_evaluation e, notes_moduleimpl mi, notes_formsemestre s
|
|
WHERE s.dept_id=%(dept_id)s
|
|
AND s.id = mi.formsemestre_id
|
|
AND mi.id = e.moduleimpl_id;
|
|
""",
|
|
{"dept_id": g.scodoc_dept_id},
|
|
)
|
|
]
|
|
cls.delete_many(evaluation_ids)
|
|
|
|
|
|
class AbsSemEtudCache(ScoDocCache):
|
|
"""Cache pour les comptes d'absences d'un étudiant dans un semestre.
|
|
Ce cache étant indépendant des semestres, le compte peut être faux lorsqu'on
|
|
change les dates début/fin d'un semestre.
|
|
C'est pourquoi il expire après timeout secondes.
|
|
Le timeout evite aussi d'éliminer explicitement ces éléments cachés lors
|
|
des suppressions d'étudiants ou de semestres.
|
|
Clé: etudid + "_" + date_debut + "_" + date_fin
|
|
Valeur: (nb_abs, nb_abs_just)
|
|
"""
|
|
|
|
prefix = "ABSE"
|
|
timeout = 60 * 60 # ttl 60 minutes
|
|
|
|
|
|
class SemBulletinsPDFCache(ScoDocCache):
|
|
"""Cache pour les classeurs de bulletins PDF d'un semestre.
|
|
Document pdf assez volumineux. La clé inclut le type de bulletin (version).
|
|
Clé: formsemestre_id_version
|
|
Valeur: (filename, pdfdoc)
|
|
"""
|
|
|
|
prefix = "SBPDF"
|
|
timeout = 12 * 60 * 60 # ttl 12h
|
|
|
|
@classmethod
|
|
def invalidate_sems(cls, formsemestre_ids):
|
|
"""Clear cached pdf for all given formsemestres"""
|
|
for version in scu.BULLETINS_VERSIONS_BUT:
|
|
oids = [
|
|
str(formsemestre_id) + "_" + version
|
|
for formsemestre_id in formsemestre_ids
|
|
]
|
|
cls.delete_many(oids)
|
|
|
|
|
|
class SemInscriptionsCache(ScoDocCache):
|
|
"""Cache les inscriptions à un semestre.
|
|
Clé: formsemestre_id
|
|
Valeur: liste d'inscriptions
|
|
[ {'formsemestre_inscription_id': 'SI78677', 'etudid': '1234', 'formsemestre_id': 'SEM012', 'etat': 'I', 'etape': ''}, ... ]
|
|
"""
|
|
|
|
prefix = "SI"
|
|
duration = 12 * 60 * 60 # ttl 12h
|
|
|
|
|
|
class TableRecapCache(ScoDocCache):
|
|
"""Cache table recap (pour TableRecap)
|
|
Clé: formsemestre_id
|
|
Valeur: le html (<div class="table_recap">...</div>)
|
|
"""
|
|
|
|
prefix = "RECAP"
|
|
duration = 12 * 60 * 60 # ttl 12h
|
|
|
|
|
|
class TableRecapWithEvalsCache(ScoDocCache):
|
|
"""Cache table recap (pour TableRecap)
|
|
Clé: formsemestre_id
|
|
Valeur: le html (<div class="table_recap">...</div>)
|
|
"""
|
|
|
|
prefix = "RECAPWITHEVALS"
|
|
duration = 12 * 60 * 60 # ttl 12h
|
|
|
|
|
|
class TableJuryCache(ScoDocCache):
|
|
"""Cache table recap (pour TableRecap)
|
|
Clé: formsemestre_id
|
|
Valeur: le html (<div class="table_recap">...</div>)
|
|
"""
|
|
|
|
prefix = "RECAPJURY"
|
|
duration = 12 * 60 * 60 # ttl 12h
|
|
|
|
|
|
class TableJuryWithEvalsCache(ScoDocCache):
|
|
"""Cache table recap (pour TableRecap)
|
|
Clé: formsemestre_id
|
|
Valeur: le html (<div class="table_recap">...</div>)
|
|
"""
|
|
|
|
prefix = "RECAPJURYWITHEVALS"
|
|
duration = 12 * 60 * 60 # ttl 12h
|
|
|
|
|
|
def invalidate_formsemestre( # was inval_cache(formsemestre_id=None, pdfonly=False)
|
|
formsemestre_id=None, pdfonly=False
|
|
):
|
|
"""expire cache pour un semestre (ou tous ceux du département si formsemestre_id non spécifié).
|
|
Si pdfonly, n'expire que les bulletins pdf cachés.
|
|
"""
|
|
from app.models.formsemestre import FormSemestre
|
|
from app.scodoc import sco_cursus
|
|
|
|
assert isinstance(formsemestre_id, int) or formsemestre_id is None
|
|
if getattr(g, "defer_cache_invalidation", 0) > 0:
|
|
g.sem_to_invalidate.add(formsemestre_id)
|
|
return
|
|
if getattr(g, "scodoc_dept") is None:
|
|
# appel via API ou tests sans dept:
|
|
formsemestre = None
|
|
if formsemestre_id:
|
|
formsemestre = db.session.get(FormSemestre, formsemestre_id)
|
|
if formsemestre is None:
|
|
raise ScoException("invalidate_formsemestre: departement must be set")
|
|
app.set_sco_dept(formsemestre.departement.acronym, open_cnx=False)
|
|
|
|
if formsemestre_id is None:
|
|
# clear all caches
|
|
log(f"--- invalidate_formsemestre: clearing all caches. pdfonly={pdfonly}---")
|
|
formsemestre_ids = [
|
|
formsemestre.id
|
|
for formsemestre in FormSemestre.query.filter_by(dept_id=g.scodoc_dept_id)
|
|
]
|
|
else:
|
|
formsemestre_ids = [
|
|
formsemestre_id
|
|
] + sco_cursus.list_formsemestre_utilisateurs_uecap(formsemestre_id)
|
|
log(
|
|
f"--- invalidate_formsemestre: clearing {formsemestre_ids}. pdfonly={pdfonly} ---"
|
|
)
|
|
|
|
if not pdfonly:
|
|
# Delete cached notes and evaluations
|
|
if formsemestre_id:
|
|
for fid in formsemestre_ids:
|
|
EvaluationCache.invalidate_sem(fid)
|
|
if (
|
|
hasattr(g, "formsemestre_results_cache")
|
|
and fid in g.formsemestre_results_cache
|
|
):
|
|
del g.formsemestre_results_cache[fid]
|
|
|
|
else:
|
|
# optimization when we invalidate all evaluations:
|
|
EvaluationCache.invalidate_all_sems()
|
|
if hasattr(g, "formsemestre_results_cache"):
|
|
del g.formsemestre_results_cache
|
|
SemInscriptionsCache.delete_many(formsemestre_ids)
|
|
ResultatsSemestreCache.delete_many(formsemestre_ids)
|
|
ValidationsSemestreCache.delete_many(formsemestre_ids)
|
|
TableRecapCache.delete_many(formsemestre_ids)
|
|
TableRecapWithEvalsCache.delete_many(formsemestre_ids)
|
|
TableJuryCache.delete_many(formsemestre_ids)
|
|
TableJuryWithEvalsCache.delete_many(formsemestre_ids)
|
|
|
|
SemBulletinsPDFCache.invalidate_sems(formsemestre_ids)
|
|
|
|
|
|
def invalidate_formsemestre_etud(etud: "Identite"):
|
|
"""Invalide tous les formsemestres auxquels l'étudiant est inscrit"""
|
|
from app.models import FormSemestre, FormSemestreInscription
|
|
|
|
inscriptions = (
|
|
FormSemestreInscription.query.filter_by(etudid=etud.id)
|
|
.join(FormSemestre)
|
|
.filter_by(dept_id=g.scodoc_dept_id)
|
|
)
|
|
for inscription in inscriptions:
|
|
invalidate_formsemestre(inscription.formsemestre_id)
|
|
|
|
|
|
class DeferredSemCacheManager:
|
|
"""Contexte pour effectuer des opérations indépendantes dans la
|
|
même requete qui invalident le cache. Par exemple, quand on inscrit
|
|
des étudiants un par un à un semestre, chaque inscription va invalider
|
|
le cache, et la suivante va le reconstruire... pour l'invalider juste après.
|
|
Ce context manager permet de grouper les invalidations.
|
|
"""
|
|
|
|
def __enter__(self):
|
|
if not hasattr(g, "defer_cache_invalidation"):
|
|
g.defer_cache_invalidation = 0
|
|
g.sem_to_invalidate = set()
|
|
g.defer_cache_invalidation += 1
|
|
return True
|
|
|
|
def __exit__(self, exc_type, exc_value, exc_traceback):
|
|
assert g.defer_cache_invalidation
|
|
g.defer_cache_invalidation -= 1
|
|
if g.defer_cache_invalidation == 0:
|
|
while g.sem_to_invalidate:
|
|
formsemestre_id = g.sem_to_invalidate.pop()
|
|
invalidate_formsemestre(formsemestre_id)
|
|
|
|
|
|
# ---- Nouvelles classes ScoDoc 9.2
|
|
class ResultatsSemestreCache(ScoDocCache):
|
|
"""Cache pour les résultats ResultatsSemestre (notes et moyennes)
|
|
Clé: formsemestre_id
|
|
Valeur: { un paquet de dataframes }
|
|
"""
|
|
|
|
prefix = "RSEM"
|
|
timeout = 60 * 60 # ttl 1 heure (en phase de mise au point)
|
|
|
|
|
|
class ValidationsSemestreCache(ScoDocCache):
|
|
"""Cache pour les résultats de jury d'un semestre
|
|
Clé: formsemestre_id
|
|
Valeur: un paquet de DataFrames
|
|
"""
|
|
|
|
prefix = "VSC"
|
|
timeout = 60 * 60 # ttl 1 heure (en phase de mise au point)
|
|
|
|
|
|
class SimpleIndexCache(ScoDocCache):
|
|
prefix = "INDEX"
|
|
|
|
|
|
class RequeteTableauAssiduiteCache(ScoDocCache):
|
|
"""
|
|
clé : "<titre_tableau>:<type_obj>:<show_pres>:<show_retard>>:<order_col>:<order>"
|
|
Valeur = liste de dicts
|
|
"""
|
|
|
|
prefix = "TABASSI"
|
|
timeout = 60 * 60 # Une heure
|
|
|
|
@classmethod
|
|
def set(cls, oid: str, value: object):
|
|
"""Ajoute une entrée au cache. Ajoute la clé dans la liste des clés du cache"""
|
|
keys_index = cls.get_index()
|
|
|
|
# On met à jour l'index
|
|
if oid not in keys_index:
|
|
keys_index.append(oid)
|
|
SimpleIndexCache.set(cls.prefix + "_index", keys_index)
|
|
|
|
# On cache la valeur
|
|
return super().set(oid, value)
|
|
|
|
@classmethod
|
|
def get_index(cls) -> list:
|
|
"""récupère la liste des clés des entrées du cache"""
|
|
# on définie un index des clés pour faciliter l'invalidation
|
|
keys_index: list = SimpleIndexCache.get(cls.prefix + "_index")
|
|
if keys_index is None:
|
|
keys_index = []
|
|
|
|
return keys_index
|
|
|
|
@classmethod
|
|
def delete_with(cls, start: str):
|
|
"""Invalide toutes les entrées de cache commençant par <start>"""
|
|
keys_index: list[str] = cls.get_index()
|
|
|
|
key: str
|
|
filtered_keys_index: list = [key for key in keys_index if key.startswith(start)]
|
|
|
|
for key in filtered_keys_index:
|
|
cls.delete(key)
|
|
|
|
SimpleIndexCache.set(
|
|
cls.prefix + "_index",
|
|
[k for k in keys_index if k not in filtered_keys_index],
|
|
)
|