# -*- mode: python -*- # -*- coding: utf-8 -*- ############################################################################## # # Gestion scolarite IUT # # Copyright (c) 1999 - 2024 Emmanuel Viennet. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # # Emmanuel Viennet emmanuel.viennet@viennet.net # ############################################################################## """Gestion des caches Ré-écrite pour ScoDoc8, utilise flask_caching et REDIS ScoDoc est maintenant multiprocessus / mono-thread, avec un cache partagé. """ # API pour les caches: # sco_cache.MyCache.get( formsemestre_id) # => sco_cache.MyCache.get(formsemestre_id) # # sco_cache.MyCache.delete(formsemestre_id) # sco_cache.MyCache.delete_many(formsemestre_id_list) # # Bulletins PDF: # sco_cache.SemBulletinsPDFCache.get(formsemestre_id, version) # sco_cache.SemBulletinsPDFCache.set(formsemestre_id, version, filename, pdfdoc) # sco_cache.SemBulletinsPDFCache.delete(formsemestre_id) suppr. toutes les versions # Evaluations: # sco_cache.EvaluationCache.get(evaluation_id), set(evaluation_id, value), delete(evaluation_id), # import traceback from flask import g import app from app import db, log from app.scodoc import notesdb as ndb from app.scodoc import sco_utils as scu from app.scodoc.sco_exceptions import ScoException CACHE = None # set in app.__init__.py class ScoDocCache: """Cache for ScoDoc objects. keys are prefixed by the current departement: g.scodoc_dept MUST be set. """ timeout = None # ttl, infinite by default prefix = "" verbose = False # if true, verbose logging (debug) @classmethod def _get_key(cls, oid): return g.scodoc_dept + "_" + cls.prefix + "_" + str(oid) @classmethod def get(cls, oid): """Returns cached object, or None""" key = cls._get_key(oid) try: return CACHE.get(key) except: log(f"XXX CACHE Warning: error in get(key={key})") log(traceback.format_exc()) return None @classmethod def set(cls, oid, value): """Store value""" key = cls._get_key(oid) if cls.verbose: log( f"{cls.__name__}.set key={key}, type={type(value).__name__}, timeout={cls.timeout}" ) try: status = CACHE.set(key, value, timeout=cls.timeout) if not status: log("Error: cache set failed !") except Exception as exc: log("XXX CACHE Warning: error in set !!!") log(exc) status = None return status @classmethod def delete(cls, oid): """Remove from cache""" # if cls.verbose: # log(f"{cls.__name__}.delete({oid})") CACHE.delete(cls._get_key(oid)) @classmethod def delete_many(cls, oids): """Remove multiple keys at once""" if cls.verbose: log(f"{cls.__name__}.delete_many({oids})") # delete_many seems bugged: # CACHE.delete_many([cls._get_key(oid) for oid in oids]) for oid in oids: cls.delete(oid) @classmethod def delete_pattern(cls, pattern: str, std_prefix=True) -> int: """Delete all keys matching pattern. The pattern starts with flask_cache_. If std_prefix is true (default), the prefix is added to the given pattern. Examples: 'TABASSI_tableau-etud-1234:*' Or, with std_prefix false, 'flask_cache_RT_TABASSI_tableau-etud-1234:*' Returns number of keys deleted. """ # see https://stackoverflow.com/questions/36708461/flask-cache-list-keys-based-on-a-pattern assert CACHE.cache.__class__.__name__ == "RedisCache" # Redis specific import redis if std_prefix: pattern = "flask_cache_" + g.scodoc_dept + "_" + cls.prefix + "_" + pattern r = redis.Redis() count = 0 for key in r.scan_iter(pattern): log(f"{cls.__name__}.delete_pattern({key})") r.delete(key) count += 1 return count class EvaluationCache(ScoDocCache): """Cache for evaluations. Clé: evaluation_id Valeur: { 'etudid' : note } """ prefix = "EVAL" @classmethod def invalidate_sem(cls, formsemestre_id): "delete evaluations in this formsemestre from cache" from app.models.evaluations import Evaluation from app.models.moduleimpls import ModuleImpl evaluation_ids = [ e.id for e in Evaluation.query.join(ModuleImpl).filter_by( formsemestre_id=formsemestre_id ) ] cls.delete_many(evaluation_ids) @classmethod def invalidate_all_sems(cls): "delete all evaluations in current dept from cache" evaluation_ids = [ x[0] for x in ndb.SimpleQuery( """SELECT e.id FROM notes_evaluation e, notes_moduleimpl mi, notes_formsemestre s WHERE s.dept_id=%(dept_id)s AND s.id = mi.formsemestre_id AND mi.id = e.moduleimpl_id; """, {"dept_id": g.scodoc_dept_id}, ) ] cls.delete_many(evaluation_ids) class AbsSemEtudCache(ScoDocCache): """Cache pour les comptes d'absences d'un étudiant dans un semestre. Ce cache étant indépendant des semestres, le compte peut être faux lorsqu'on change les dates début/fin d'un semestre. C'est pourquoi il expire après timeout secondes. Le timeout evite aussi d'éliminer explicitement ces éléments cachés lors des suppressions d'étudiants ou de semestres. Clé: etudid + "_" + date_debut + "_" + date_fin Valeur: (nb_abs, nb_abs_just) """ prefix = "ABSE" timeout = 60 * 60 # ttl 60 minutes class SemBulletinsPDFCache(ScoDocCache): """Cache pour les classeurs de bulletins PDF d'un semestre. Document pdf assez volumineux. La clé inclut le type de bulletin (version). Clé: formsemestre_id_version Valeur: (filename, pdfdoc) """ prefix = "SBPDF" timeout = 12 * 60 * 60 # ttl 12h @classmethod def invalidate_sems(cls, formsemestre_ids): """Clear cached pdf for all given formsemestres""" for version in scu.BULLETINS_VERSIONS_BUT: oids = [ str(formsemestre_id) + "_" + version for formsemestre_id in formsemestre_ids ] cls.delete_many(oids) class SemInscriptionsCache(ScoDocCache): """Cache les inscriptions à un semestre. Clé: formsemestre_id Valeur: liste d'inscriptions [ {'formsemestre_inscription_id': 'SI78677', 'etudid': '1234', 'formsemestre_id': 'SEM012', 'etat': 'I', 'etape': ''}, ... ] """ prefix = "SI" duration = 12 * 60 * 60 # ttl 12h class TableRecapCache(ScoDocCache): """Cache table recap (pour TableRecap) Clé: formsemestre_id Valeur: le html (
...
) """ prefix = "RECAP" duration = 12 * 60 * 60 # ttl 12h class TableRecapWithEvalsCache(ScoDocCache): """Cache table recap (pour TableRecap) Clé: formsemestre_id Valeur: le html (
...
) """ prefix = "RECAPWITHEVALS" duration = 12 * 60 * 60 # ttl 12h class TableJuryCache(ScoDocCache): """Cache table recap (pour TableRecap) Clé: formsemestre_id Valeur: le html (
...
) """ prefix = "RECAPJURY" duration = 12 * 60 * 60 # ttl 12h class TableJuryWithEvalsCache(ScoDocCache): """Cache table recap (pour TableRecap) Clé: formsemestre_id Valeur: le html (
...
) """ prefix = "RECAPJURYWITHEVALS" duration = 12 * 60 * 60 # ttl 12h def invalidate_formsemestre( # was inval_cache(formsemestre_id=None, pdfonly=False) formsemestre_id=None, pdfonly=False ): """expire cache pour un semestre (ou tous ceux du département si formsemestre_id non spécifié). Si pdfonly, n'expire que les bulletins pdf cachés. """ from app.models.formsemestre import FormSemestre from app.scodoc import sco_cursus assert isinstance(formsemestre_id, int) or formsemestre_id is None if getattr(g, "defer_cache_invalidation", 0) > 0: g.sem_to_invalidate.add(formsemestre_id) return if getattr(g, "scodoc_dept") is None: # appel via API ou tests sans dept: formsemestre = None if formsemestre_id: formsemestre = db.session.get(FormSemestre, formsemestre_id) if formsemestre is None: raise ScoException("invalidate_formsemestre: departement must be set") app.set_sco_dept(formsemestre.departement.acronym, open_cnx=False) if formsemestre_id is None: # clear all caches log(f"--- invalidate_formsemestre: clearing all caches. pdfonly={pdfonly}---") formsemestre_ids = [ formsemestre.id for formsemestre in FormSemestre.query.filter_by(dept_id=g.scodoc_dept_id) ] else: formsemestre_ids = [ formsemestre_id ] + sco_cursus.list_formsemestre_utilisateurs_uecap(formsemestre_id) log( f"--- invalidate_formsemestre: clearing {formsemestre_ids}. pdfonly={pdfonly} ---" ) if not pdfonly: # Delete cached notes and evaluations if formsemestre_id: for fid in formsemestre_ids: EvaluationCache.invalidate_sem(fid) if ( hasattr(g, "formsemestre_results_cache") and fid in g.formsemestre_results_cache ): del g.formsemestre_results_cache[fid] else: # optimization when we invalidate all evaluations: EvaluationCache.invalidate_all_sems() if hasattr(g, "formsemestre_results_cache"): del g.formsemestre_results_cache SemInscriptionsCache.delete_many(formsemestre_ids) ResultatsSemestreCache.delete_many(formsemestre_ids) ValidationsSemestreCache.delete_many(formsemestre_ids) TableRecapCache.delete_many(formsemestre_ids) TableRecapWithEvalsCache.delete_many(formsemestre_ids) TableJuryCache.delete_many(formsemestre_ids) TableJuryWithEvalsCache.delete_many(formsemestre_ids) SemBulletinsPDFCache.invalidate_sems(formsemestre_ids) def invalidate_formsemestre_etud(etud: "Identite"): """Invalide tous les formsemestres auxquels l'étudiant est inscrit""" from app.models import FormSemestre, FormSemestreInscription inscriptions = ( FormSemestreInscription.query.filter_by(etudid=etud.id) .join(FormSemestre) .filter_by(dept_id=g.scodoc_dept_id) ) for inscription in inscriptions: invalidate_formsemestre(inscription.formsemestre_id) class DeferredSemCacheManager: """Contexte pour effectuer des opérations indépendantes dans la même requete qui invalident le cache. Par exemple, quand on inscrit des étudiants un par un à un semestre, chaque inscription va invalider le cache, et la suivante va le reconstruire... pour l'invalider juste après. Ce context manager permet de grouper les invalidations. """ def __enter__(self): if not hasattr(g, "defer_cache_invalidation"): g.defer_cache_invalidation = 0 g.sem_to_invalidate = set() g.defer_cache_invalidation += 1 return True def __exit__(self, exc_type, exc_value, exc_traceback): assert g.defer_cache_invalidation g.defer_cache_invalidation -= 1 if g.defer_cache_invalidation == 0: while g.sem_to_invalidate: formsemestre_id = g.sem_to_invalidate.pop() invalidate_formsemestre(formsemestre_id) # ---- Nouvelles classes ScoDoc 9.2 class ResultatsSemestreCache(ScoDocCache): """Cache pour les résultats ResultatsSemestre (notes et moyennes) Clé: formsemestre_id Valeur: { un paquet de dataframes } """ prefix = "RSEM" timeout = 60 * 60 # ttl 1 heure (en phase de mise au point) class ValidationsSemestreCache(ScoDocCache): """Cache pour les résultats de jury d'un semestre Clé: formsemestre_id Valeur: un paquet de DataFrames """ prefix = "VSC" timeout = 60 * 60 # ttl 1 heure (en phase de mise au point) class RequeteTableauAssiduiteCache(ScoDocCache): """ clé : ":::>::" Valeur = liste de dicts """ prefix = "TABASSI" timeout = 60 * 60 # Une heure