ScoDoc-Lille/app/scodoc/sco_cache.py

# -*- mode: python -*-
# -*- coding: utf-8 -*-

##############################################################################
#
# Gestion scolarite IUT
#
# Copyright (c) 1999 - 2024 Emmanuel Viennet.  All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#   Emmanuel Viennet      emmanuel.viennet@viennet.net
#
##############################################################################

"""Gestion des caches

  Ré-écrite pour ScoDoc8, utilise flask_caching et REDIS

  ScoDoc est maintenant multiprocessus / mono-thread, avec un cache partagé.
"""


# API pour les caches:
#  sco_cache.MyCache.get( formsemestre_id)
#   => sco_cache.MyCache.get(formsemestre_id)
#
#  sco_cache.MyCache.delete(formsemestre_id)
#  sco_cache.MyCache.delete_many(formsemestre_id_list)
#
# Bulletins PDF:
#  sco_cache.SemBulletinsPDFCache.get(formsemestre_id, version)
#  sco_cache.SemBulletinsPDFCache.set(formsemestre_id, version, filename, pdfdoc)
#  sco_cache.SemBulletinsPDFCache.delete(formsemestre_id) suppr. toutes les versions

# Evaluations:
#  sco_cache.EvaluationCache.get(evaluation_id), set(evaluation_id, value), delete(evaluation_id),
#

import traceback

from flask import g

import app
from app import db, log
from app.scodoc import notesdb as ndb
from app.scodoc import sco_utils as scu
from app.scodoc.sco_exceptions import ScoException

CACHE = None  # set in app.__init__.py


class ScoDocCache:
    """Cache for ScoDoc objects.
    keys are prefixed by the current departement: g.scodoc_dept MUST be set.
    """

    timeout = None  # ttl, infinite by default
    prefix = ""
    verbose = False  # if true, verbose logging (debug)

    @classmethod
    def _get_key(cls, oid):
        return g.scodoc_dept + "_" + cls.prefix + "_" + str(oid)

    @classmethod
    def get(cls, oid):
        """Returns cached object, or None"""
        key = cls._get_key(oid)
        try:
            return CACHE.get(key)
        except:
            log(f"XXX CACHE Warning: error in get(key={key})")
            log(traceback.format_exc())
            return None

    @classmethod
    def set(cls, oid, value):
        """Store value"""
        key = cls._get_key(oid)
        if cls.verbose:
            log(
                f"{cls.__name__}.set key={key}, type={type(value).__name__}, timeout={cls.timeout}"
            )
        try:
            status = CACHE.set(key, value, timeout=cls.timeout)
            if not status:
                log("Error: cache set failed !")
        except Exception as exc:
            log("XXX CACHE Warning: error in set !!!")
            log(exc)
            status = None
        return status

    @classmethod
    def delete(cls, oid):
        """Remove from cache"""
        # if cls.verbose:
        #    log(f"{cls.__name__}.delete({oid})")
        CACHE.delete(cls._get_key(oid))

    @classmethod
    def delete_many(cls, oids):
        """Remove multiple keys at once"""
        if cls.verbose:
            log(f"{cls.__name__}.delete_many({oids})")
        # delete_many seems bugged:
        # CACHE.delete_many([cls._get_key(oid) for oid in oids])
        for oid in oids:
            cls.delete(oid)

    @classmethod
    def delete_pattern(cls, pattern: str, std_prefix=True) -> int:
        """Delete all keys matching pattern.
        The pattern starts with flask_cache_<dept_acronym>.
        If std_prefix is true (default), the prefix is added
        to the given pattern.
        Examples:
        'TABASSI_tableau-etud-1234:*'
        Or, with std_prefix false, 'flask_cache_RT_TABASSI_tableau-etud-1234:*'

        Returns number of keys deleted.
        """
        # see https://stackoverflow.com/questions/36708461/flask-cache-list-keys-based-on-a-pattern
        assert CACHE.cache.__class__.__name__ == "RedisCache"  # Redis specific
        import redis

        if std_prefix:
            pattern = "flask_cache_" + g.scodoc_dept + "_" + cls.prefix + "_" + pattern

        r = redis.Redis()
        count = 0
        for key in r.scan_iter(pattern):
            log(f"{cls.__name__}.delete_pattern({key})")
            r.delete(key)
            count += 1
        return count


class EvaluationCache(ScoDocCache):
    """Cache for evaluations.
    Clé: evaluation_id
    Valeur: { 'etudid' : note }
    """

    prefix = "EVAL"

    @classmethod
    def invalidate_sem(cls, formsemestre_id):
        "delete evaluations in this formsemestre from cache"
        from app.models.evaluations import Evaluation
        from app.models.moduleimpls import ModuleImpl

        evaluation_ids = [
            e.id
            for e in Evaluation.query.join(ModuleImpl).filter_by(
                formsemestre_id=formsemestre_id
            )
        ]
        cls.delete_many(evaluation_ids)

    @classmethod
    def invalidate_all_sems(cls):
        "delete all evaluations in current dept from cache"
        evaluation_ids = [
            x[0]
            for x in ndb.SimpleQuery(
                """SELECT e.id
                FROM notes_evaluation e, notes_moduleimpl mi, notes_formsemestre s
                WHERE s.dept_id=%(dept_id)s
                AND s.id = mi.formsemestre_id
                AND mi.id = e.moduleimpl_id;
                """,
                {"dept_id": g.scodoc_dept_id},
            )
        ]
        cls.delete_many(evaluation_ids)


class AbsSemEtudCache(ScoDocCache):
    """Cache pour les comptes d'absences d'un étudiant dans un semestre.
    Ce cache étant indépendant des semestres, le compte peut être faux lorsqu'on
    change les dates début/fin d'un semestre.
    C'est pourquoi il expire après timeout secondes.
    Le timeout evite aussi d'éliminer explicitement ces éléments cachés lors
    des suppressions d'étudiants ou de semestres.
    Clé: etudid + "_" + date_debut + "_" + date_fin
    Valeur: (nb_abs, nb_abs_just)
    """

    prefix = "ABSE"
    timeout = 60 * 60  # ttl 60 minutes


class SemBulletinsPDFCache(ScoDocCache):
    """Cache pour les classeurs de bulletins PDF d'un semestre.
    Document pdf assez volumineux. La clé inclut le type de bulletin (version).
    Clé: formsemestre_id_version
    Valeur: (filename, pdfdoc)
    """

    prefix = "SBPDF"
    timeout = 12 * 60 * 60  # ttl 12h

    @classmethod
    def invalidate_sems(cls, formsemestre_ids):
        """Clear cached pdf for all given formsemestres"""
        for version in scu.BULLETINS_VERSIONS_BUT:
            oids = [
                str(formsemestre_id) + "_" + version
                for formsemestre_id in formsemestre_ids
            ]
            cls.delete_many(oids)


class SemInscriptionsCache(ScoDocCache):
    """Cache les inscriptions à un semestre.
    Clé: formsemestre_id
    Valeur: liste d'inscriptions
    [ {'formsemestre_inscription_id': 'SI78677', 'etudid': '1234', 'formsemestre_id': 'SEM012', 'etat': 'I', 'etape': ''}, ... ]
    """

    prefix = "SI"
    duration = 12 * 60 * 60  # ttl 12h


class TableRecapCache(ScoDocCache):
    """Cache table recap (pour TableRecap)
    Clé: formsemestre_id
    Valeur: le html (<div class="table_recap">...</div>)
    """

    prefix = "RECAP"
    duration = 12 * 60 * 60  # ttl 12h


class TableRecapWithEvalsCache(ScoDocCache):
    """Cache table recap (pour TableRecap)
    Clé: formsemestre_id
    Valeur: le html (<div class="table_recap">...</div>)
    """

    prefix = "RECAPWITHEVALS"
    duration = 12 * 60 * 60  # ttl 12h


class TableJuryCache(ScoDocCache):
    """Cache table recap (pour TableRecap)
    Clé: formsemestre_id
    Valeur: le html (<div class="table_recap">...</div>)
    """

    prefix = "RECAPJURY"
    duration = 12 * 60 * 60  # ttl 12h


class TableJuryWithEvalsCache(ScoDocCache):
    """Cache table recap (pour TableRecap)
    Clé: formsemestre_id
    Valeur: le html (<div class="table_recap">...</div>)
    """

    prefix = "RECAPJURYWITHEVALS"
    duration = 12 * 60 * 60  # ttl 12h


def invalidate_formsemestre(  # was inval_cache(formsemestre_id=None, pdfonly=False)
    formsemestre_id=None, pdfonly=False
):
    """expire cache pour un semestre (ou tous ceux du département si formsemestre_id non spécifié).
    Si pdfonly, n'expire que les bulletins pdf cachés.
    """
    from app.models.formsemestre import FormSemestre
    from app.scodoc import sco_cursus

    assert isinstance(formsemestre_id, int) or formsemestre_id is None
    if getattr(g, "defer_cache_invalidation", 0) > 0:
        g.sem_to_invalidate.add(formsemestre_id)
        return
    if getattr(g, "scodoc_dept") is None:
        # appel via API ou tests sans dept:
        formsemestre = None
        if formsemestre_id:
            formsemestre = db.session.get(FormSemestre, formsemestre_id)
        if formsemestre is None:
            raise ScoException("invalidate_formsemestre: departement must be set")
        app.set_sco_dept(formsemestre.departement.acronym, open_cnx=False)

    if formsemestre_id is None:
        # clear all caches
        log(f"--- invalidate_formsemestre: clearing all caches. pdfonly={pdfonly}---")
        formsemestre_ids = [
            formsemestre.id
            for formsemestre in FormSemestre.query.filter_by(dept_id=g.scodoc_dept_id)
        ]
    else:
        formsemestre_ids = [
            formsemestre_id
        ] + sco_cursus.list_formsemestre_utilisateurs_uecap(formsemestre_id)
        log(
            f"--- invalidate_formsemestre: clearing {formsemestre_ids}. pdfonly={pdfonly} ---"
        )

    if not pdfonly:
        # Delete cached notes and evaluations
        if formsemestre_id:
            for fid in formsemestre_ids:
                EvaluationCache.invalidate_sem(fid)
                if (
                    hasattr(g, "formsemestre_results_cache")
                    and fid in g.formsemestre_results_cache
                ):
                    del g.formsemestre_results_cache[fid]

        else:
            # optimization when we invalidate all evaluations:
            EvaluationCache.invalidate_all_sems()
            if hasattr(g, "formsemestre_results_cache"):
                del g.formsemestre_results_cache
        SemInscriptionsCache.delete_many(formsemestre_ids)
        ResultatsSemestreCache.delete_many(formsemestre_ids)
        ValidationsSemestreCache.delete_many(formsemestre_ids)
        TableRecapCache.delete_many(formsemestre_ids)
        TableRecapWithEvalsCache.delete_many(formsemestre_ids)
        TableJuryCache.delete_many(formsemestre_ids)
        TableJuryWithEvalsCache.delete_many(formsemestre_ids)

    SemBulletinsPDFCache.invalidate_sems(formsemestre_ids)


def invalidate_formsemestre_etud(etud: "Identite"):
    """Invalide tous les formsemestres auxquels l'étudiant est inscrit"""
    from app.models import FormSemestre, FormSemestreInscription

    inscriptions = (
        FormSemestreInscription.query.filter_by(etudid=etud.id)
        .join(FormSemestre)
        .filter_by(dept_id=g.scodoc_dept_id)
    )
    for inscription in inscriptions:
        invalidate_formsemestre(inscription.formsemestre_id)


class DeferredSemCacheManager:
    """Contexte pour effectuer des opérations indépendantes dans la
    même requete qui invalident le cache. Par exemple, quand on inscrit
    des étudiants un par un à un semestre, chaque inscription va invalider
    le cache, et la suivante va le reconstruire... pour l'invalider juste après.
    Ce context manager permet de grouper les invalidations.
    """

    def __enter__(self):
        if not hasattr(g, "defer_cache_invalidation"):
            g.defer_cache_invalidation = 0
            g.sem_to_invalidate = set()
        g.defer_cache_invalidation += 1
        return True

    def __exit__(self, exc_type, exc_value, exc_traceback):
        assert g.defer_cache_invalidation
        g.defer_cache_invalidation -= 1
        if g.defer_cache_invalidation == 0:
            while g.sem_to_invalidate:
                formsemestre_id = g.sem_to_invalidate.pop()
                invalidate_formsemestre(formsemestre_id)


# ---- Nouvelles classes ScoDoc 9.2
class ResultatsSemestreCache(ScoDocCache):
    """Cache pour les résultats ResultatsSemestre (notes et moyennes)
    Clé: formsemestre_id
    Valeur: { un paquet de dataframes }
    """

    prefix = "RSEM"
    timeout = 60 * 60  # ttl 1 heure (en phase de mise au point)


class ValidationsSemestreCache(ScoDocCache):
    """Cache pour les résultats de jury d'un semestre
    Clé: formsemestre_id
    Valeur: un paquet de DataFrames
    """

    prefix = "VSC"
    timeout = 60 * 60  # ttl 1 heure (en phase de mise au point)


class SimpleIndexCache(ScoDocCache):
    prefix = "INDEX"


class RequeteTableauAssiduiteCache(ScoDocCache):
    """
    clé : "<titre_tableau>:<type_obj>:<show_pres>:<show_retard>>:<order_col>:<order>"
    Valeur = liste de dicts
    """

    prefix = "TABASSI"
    timeout = 60 * 60  # Une heure

    @classmethod
    def set(cls, oid: str, value: object):
        """Ajoute une entrée au cache. Ajoute la clé dans la liste des clés du cache"""
        keys_index = cls.get_index()

        # On met à jour l'index
        if oid not in keys_index:
            keys_index.append(oid)
            SimpleIndexCache.set(cls.prefix + "_index", keys_index)

        # On cache la valeur
        return super().set(oid, value)

    @classmethod
    def get_index(cls) -> list:
        """récupère la liste des clés des entrées du cache"""
        # on définie un index des clés pour faciliter l'invalidation
        keys_index: list = SimpleIndexCache.get(cls.prefix + "_index")
        if keys_index is None:
            keys_index = []

        return keys_index

    @classmethod
    def delete_with(cls, start: str):
        """Invalide toutes les entrées de cache commençant par <start>"""
        keys_index: list[str] = cls.get_index()

        key: str
        filtered_keys_index: list = [key for key in keys_index if key.startswith(start)]

        for key in filtered_keys_index:
            cls.delete(key)

        SimpleIndexCache.set(
            cls.prefix + "_index",
            [k for k in keys_index if k not in filtered_keys_index],
        )