ScoDoc/app/scodoc/sco_cache.py

361 lines
12 KiB
Python
Raw Permalink Normal View History

2020-09-26 16:19:37 +02:00
# -*- mode: python -*-
# -*- coding: utf-8 -*-
##############################################################################
#
# Gestion scolarite IUT
#
2023-01-02 13:16:27 +01:00
# Copyright (c) 1999 - 2023 Emmanuel Viennet. All rights reserved.
2020-09-26 16:19:37 +02:00
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# Emmanuel Viennet emmanuel.viennet@viennet.net
#
##############################################################################
2021-07-15 15:05:54 +02:00
"""Gestion des caches
2020-09-26 16:19:37 +02:00
2021-07-27 14:33:11 +02:00
-écrite pour ScoDoc8, utilise flask_caching et REDIS
2021-07-15 15:05:54 +02:00
ScoDoc est maintenant multiprocessus / mono-thread, avec un cache partagé.
2020-09-26 16:19:37 +02:00
"""
2022-02-28 16:25:18 +01:00
# API pour les caches:
# sco_cache.MyCache.get( formsemestre_id)
# => sco_cache.MyCache.get(formsemestre_id)
2021-07-15 15:05:54 +02:00
#
2022-02-28 16:25:18 +01:00
# sco_cache.MyCache.delete(formsemestre_id)
# sco_cache.MyCache.delete_many(formsemestre_id_list)
2021-07-15 15:05:54 +02:00
#
# Bulletins PDF:
2021-11-02 15:49:12 +01:00
# sco_cache.SemBulletinsPDFCache.get(formsemestre_id, version)
# sco_cache.SemBulletinsPDFCache.set(formsemestre_id, version, filename, pdfdoc)
# sco_cache.SemBulletinsPDFCache.delete(formsemestre_id) suppr. toutes les versions
2021-07-15 15:05:54 +02:00
# Evaluations:
# sco_cache.EvaluationCache.get(evaluation_id), set(evaluation_id, value), delete(evaluation_id),
#
2021-07-26 15:18:16 +02:00
import traceback
2021-07-15 15:05:54 +02:00
from flask import g
2022-07-26 09:00:48 +02:00
import app
2022-02-06 16:09:17 +01:00
from app import log
2021-07-19 19:53:01 +02:00
from app.scodoc import notesdb as ndb
from app.scodoc import sco_utils as scu
2022-07-26 09:00:48 +02:00
from app.scodoc.sco_exceptions import ScoException
2021-07-15 15:05:54 +02:00
2021-07-19 19:53:01 +02:00
CACHE = None # set in app.__init__.py
2021-07-15 15:05:54 +02:00
class ScoDocCache:
"""Cache for ScoDoc objects.
2022-07-26 09:00:48 +02:00
keys are prefixed by the current departement: g.scodoc_dept MUST be set.
2021-07-15 15:05:54 +02:00
"""
2021-07-19 19:53:01 +02:00
timeout = None # ttl, infinite by default
2021-07-15 15:05:54 +02:00
prefix = ""
verbose = False # if true, verbose logging (debug)
2021-07-15 15:05:54 +02:00
@classmethod
def _get_key(cls, oid):
2021-08-10 09:10:36 +02:00
return g.scodoc_dept + "_" + cls.prefix + "_" + str(oid)
2021-07-15 15:05:54 +02:00
@classmethod
def get(cls, oid):
2021-08-10 13:20:35 +02:00
"""Returns cached object, or None"""
2021-08-15 10:42:08 +02:00
key = cls._get_key(oid)
2021-07-26 15:18:16 +02:00
try:
2021-08-15 10:42:08 +02:00
return CACHE.get(key)
2021-07-27 14:33:11 +02:00
except:
2021-08-15 10:42:08 +02:00
log(f"XXX CACHE Warning: error in get(key={key})")
2021-07-26 15:18:16 +02:00
log(traceback.format_exc())
return None
2021-07-15 15:05:54 +02:00
@classmethod
def set(cls, oid, value):
"""Store value"""
key = cls._get_key(oid)
if cls.verbose:
log(
f"{cls.__name__}.set key={key}, type={type(value).__name__}, timeout={cls.timeout}"
)
2021-07-26 17:11:45 +02:00
try:
status = CACHE.set(key, value, timeout=cls.timeout)
if not status:
log("Error: cache set failed !")
2022-01-25 10:45:13 +01:00
except Exception as exc:
2021-07-27 14:33:11 +02:00
log("XXX CACHE Warning: error in set !!!")
2022-01-25 10:45:13 +01:00
log(exc)
2021-09-13 22:10:01 +02:00
status = None
return status
2021-07-20 06:52:42 +02:00
2021-07-15 15:05:54 +02:00
@classmethod
def delete(cls, oid):
"""Remove from cache"""
# if cls.verbose:
# log(f"{cls.__name__}.delete({oid})")
2021-07-15 15:05:54 +02:00
CACHE.delete(cls._get_key(oid))
@classmethod
def delete_many(cls, oids):
"""Remove multiple keys at once"""
if cls.verbose:
log(f"{cls.__name__}.delete_many({oids})")
# delete_many seems bugged:
# CACHE.delete_many([cls._get_key(oid) for oid in oids])
for oid in oids:
cls.delete(oid)
2021-07-15 15:05:54 +02:00
class EvaluationCache(ScoDocCache):
2021-07-21 14:58:49 +02:00
"""Cache for evaluations.
Clé: evaluation_id
Valeur: { 'etudid' : note }
"""
2021-07-15 15:05:54 +02:00
prefix = "EVAL"
@classmethod
2021-07-19 19:53:01 +02:00
def invalidate_sem(cls, formsemestre_id):
"delete evaluations in this formsemestre from cache"
2022-07-20 22:03:29 +02:00
from app.models.evaluations import Evaluation
from app.models.moduleimpls import ModuleImpl
2021-07-19 19:53:01 +02:00
evaluation_ids = [
2022-07-20 22:03:29 +02:00
e.id
for e in Evaluation.query.join(ModuleImpl).filter_by(
formsemestre_id=formsemestre_id
)
2021-07-19 19:53:01 +02:00
]
cls.delete_many(evaluation_ids)
2021-07-15 15:05:54 +02:00
@classmethod
2021-07-19 19:53:01 +02:00
def invalidate_all_sems(cls):
2021-08-13 00:34:58 +02:00
"delete all evaluations in current dept from cache"
2021-07-19 19:53:01 +02:00
evaluation_ids = [
2021-08-13 00:34:58 +02:00
x[0]
for x in ndb.SimpleQuery(
"""SELECT e.id
FROM notes_evaluation e, notes_moduleimpl mi, notes_formsemestre s
WHERE s.dept_id=%(dept_id)s
AND s.id = mi.formsemestre_id
AND mi.id = e.moduleimpl_id;
""",
{"dept_id": g.scodoc_dept_id},
)
2021-07-19 19:53:01 +02:00
]
cls.delete_many(evaluation_ids)
class AbsSemEtudCache(ScoDocCache):
"""Cache pour les comptes d'absences d'un étudiant dans un semestre.
2021-09-28 09:14:04 +02:00
Ce cache étant indépendant des semestres, le compte peut être faux lorsqu'on
2021-07-19 19:53:01 +02:00
change les dates début/fin d'un semestre.
C'est pourquoi il expire après timeout secondes.
Le timeout evite aussi d'éliminer explicitement ces éléments cachés lors
des suppressions d'étudiants ou de semestres.
2021-07-20 06:52:42 +02:00
Clé: etudid + "_" + date_debut + "_" + date_fin
Valeur: (nb_abs, nb_abs_just)
2021-07-19 19:53:01 +02:00
"""
2021-07-15 15:05:54 +02:00
2021-07-20 06:52:42 +02:00
prefix = "ABSE"
2021-07-19 19:53:01 +02:00
timeout = 60 * 60 # ttl 60 minutes
2021-07-15 15:05:54 +02:00
2021-07-19 19:53:01 +02:00
class SemBulletinsPDFCache(ScoDocCache):
"""Cache pour les classeurs de bulletins PDF d'un semestre.
Document pdf assez volumineux. La clé inclut le type de bulletin (version).
Clé: formsemestre_id_version
Valeur: (filename, pdfdoc)
"""
2021-07-15 15:05:54 +02:00
2021-07-19 19:53:01 +02:00
prefix = "SBPDF"
timeout = 12 * 60 * 60 # ttl 12h
2020-09-26 16:19:37 +02:00
2021-07-19 19:53:01 +02:00
@classmethod
def invalidate_sems(cls, formsemestre_ids):
"""Clear cached pdf for all given formsemestres"""
for version in scu.BULLETINS_VERSIONS:
oids = [
2021-08-10 09:10:36 +02:00
str(formsemestre_id) + "_" + version
for formsemestre_id in formsemestre_ids
2021-07-19 19:53:01 +02:00
]
cls.delete_many(oids)
class SemInscriptionsCache(ScoDocCache):
"""Cache les inscriptions à un semestre.
Clé: formsemestre_id
Valeur: liste d'inscriptions
[ {'formsemestre_inscription_id': 'SI78677', 'etudid': '1234', 'formsemestre_id': 'SEM012', 'etat': 'I', 'etape': ''}, ... ]
"""
2020-09-26 16:19:37 +02:00
2021-07-19 19:53:01 +02:00
prefix = "SI"
duration = 12 * 60 * 60 # ttl 12h
2020-09-26 16:19:37 +02:00
2022-04-05 22:23:55 +02:00
class TableRecapCache(ScoDocCache):
2023-02-03 22:39:45 +01:00
"""Cache table recap (pour TableRecap)
2022-04-05 22:23:55 +02:00
Clé: formsemestre_id
Valeur: le html (<div class="table_recap">...</div>)
"""
prefix = "RECAP"
duration = 12 * 60 * 60 # ttl 12h
class TableRecapWithEvalsCache(ScoDocCache):
2023-02-03 22:39:45 +01:00
"""Cache table recap (pour TableRecap)
2022-04-05 22:23:55 +02:00
Clé: formsemestre_id
Valeur: le html (<div class="table_recap">...</div>)
"""
prefix = "RECAPWITHEVALS"
duration = 12 * 60 * 60 # ttl 12h
class TableJuryCache(ScoDocCache):
"""Cache table recap (pour TableRecap)
Clé: formsemestre_id
Valeur: le html (<div class="table_recap">...</div>)
"""
prefix = "RECAPJURY"
duration = 12 * 60 * 60 # ttl 12h
class TableJuryWithEvalsCache(ScoDocCache):
"""Cache table recap (pour TableRecap)
Clé: formsemestre_id
Valeur: le html (<div class="table_recap">...</div>)
"""
prefix = "RECAPJURYWITHEVALS"
duration = 12 * 60 * 60 # ttl 12h
def invalidate_formsemestre( # was inval_cache(formsemestre_id=None, pdfonly=False)
2021-07-19 19:53:01 +02:00
formsemestre_id=None, pdfonly=False
):
"""expire cache pour un semestre (ou tous ceux du département si formsemestre_id non spécifié).
2021-07-19 19:53:01 +02:00
Si pdfonly, n'expire que les bulletins pdf cachés.
2020-09-26 16:19:37 +02:00
"""
2022-07-11 18:22:05 +02:00
from app.models.formsemestre import FormSemestre
2022-07-07 16:24:52 +02:00
from app.scodoc import sco_cursus
2021-07-19 19:53:01 +02:00
2022-07-11 18:22:05 +02:00
assert isinstance(formsemestre_id, int) or formsemestre_id is None
if getattr(g, "defer_cache_invalidation", 0) > 0:
2021-09-02 18:05:22 +02:00
g.sem_to_invalidate.add(formsemestre_id)
return
2022-07-26 09:00:48 +02:00
if getattr(g, "scodoc_dept") is None:
# appel via API ou tests sans dept:
formsemestre = None
if formsemestre_id:
formsemestre = FormSemestre.query.get(formsemestre_id)
if formsemestre is None:
raise ScoException("invalidate_formsemestre: departement must be set")
app.set_sco_dept(formsemestre.departement.acronym, open_cnx=False)
2021-07-19 19:53:01 +02:00
if formsemestre_id is None:
# clear all caches
2022-07-11 18:22:05 +02:00
log(
f"----- invalidate_formsemestre: clearing all caches. pdfonly={pdfonly}-----"
)
2021-07-19 19:53:01 +02:00
formsemestre_ids = [
2022-07-11 18:22:05 +02:00
formsemestre.id
for formsemestre in FormSemestre.query.filter_by(dept_id=g.scodoc_dept_id)
2021-07-19 19:53:01 +02:00
]
else:
formsemestre_ids = [
formsemestre_id
2022-07-07 16:24:52 +02:00
] + sco_cursus.list_formsemestre_utilisateurs_uecap(formsemestre_id)
2022-07-11 18:22:05 +02:00
log(
f"----- invalidate_formsemestre: clearing {formsemestre_ids}. pdfonly={pdfonly} -----"
)
2021-07-19 19:53:01 +02:00
if not pdfonly:
# Delete cached notes and evaluations
if formsemestre_id:
2021-07-27 19:36:10 +02:00
for fid in formsemestre_ids:
EvaluationCache.invalidate_sem(fid)
2022-02-28 16:25:18 +01:00
if (
hasattr(g, "formsemestre_results_cache")
and fid in g.formsemestre_results_cache
):
del g.formsemestre_results_cache[fid]
2020-09-26 16:19:37 +02:00
else:
2021-07-19 19:53:01 +02:00
# optimization when we invalidate all evaluations:
EvaluationCache.invalidate_all_sems()
2022-02-28 16:25:18 +01:00
if hasattr(g, "formsemestre_results_cache"):
del g.formsemestre_results_cache
2021-07-19 19:53:01 +02:00
SemInscriptionsCache.delete_many(formsemestre_ids)
2022-02-28 16:25:18 +01:00
ResultatsSemestreCache.delete_many(formsemestre_ids)
ValidationsSemestreCache.delete_many(formsemestre_ids)
2022-04-05 22:23:55 +02:00
TableRecapCache.delete_many(formsemestre_ids)
TableRecapWithEvalsCache.delete_many(formsemestre_ids)
2023-02-22 18:14:33 +01:00
TableJuryCache.delete_many(formsemestre_ids)
TableJuryWithEvalsCache.delete_many(formsemestre_ids)
2022-04-05 22:23:55 +02:00
2021-07-19 19:53:01 +02:00
SemBulletinsPDFCache.invalidate_sems(formsemestre_ids)
2021-09-02 18:05:22 +02:00
2022-05-04 20:26:45 +02:00
class DeferredSemCacheManager:
"""Contexte pour effectuer des opérations indépendantes dans la
2021-09-02 18:05:22 +02:00
même requete qui invalident le cache. Par exemple, quand on inscrit
des étudiants un par un à un semestre, chaque inscription va invalider
le cache, et la suivante va le reconstruire... pour l'invalider juste après.
Ce context manager permet de grouper les invalidations.
"""
def __enter__(self):
2022-07-11 18:22:05 +02:00
if not hasattr(g, "defer_cache_invalidation"):
g.defer_cache_invalidation = 0
g.sem_to_invalidate = set()
g.defer_cache_invalidation += 1
2021-09-02 18:05:22 +02:00
return True
def __exit__(self, exc_type, exc_value, exc_traceback):
assert g.defer_cache_invalidation
2022-07-11 18:22:05 +02:00
g.defer_cache_invalidation -= 1
if g.defer_cache_invalidation == 0:
while g.sem_to_invalidate:
formsemestre_id = g.sem_to_invalidate.pop()
invalidate_formsemestre(formsemestre_id)
2022-01-07 15:08:45 +01:00
# ---- Nouvelles classes ScoDoc 9.2
class ResultatsSemestreCache(ScoDocCache):
2022-02-06 16:09:17 +01:00
"""Cache pour les résultats ResultatsSemestre (notes et moyennes)
2022-01-07 15:08:45 +01:00
Clé: formsemestre_id
Valeur: { un paquet de dataframes }
"""
prefix = "RSEM"
timeout = 60 * 60 # ttl 1 heure (en phase de mise au point)
2022-02-06 16:09:17 +01:00
class ValidationsSemestreCache(ScoDocCache):
"""Cache pour les résultats de jury d'un semestre
Clé: formsemestre_id
Valeur: un paquet de DataFrames
"""
prefix = "VSC"
timeout = 60 * 60 # ttl 1 heure (en phase de mise au point)