From 65cdea0c76b5f92f3ef03be807e5efe8fc5e4ba5 Mon Sep 17 00:00:00 2001 From: Emmanuel Viennet Date: Thu, 15 Jul 2021 15:05:54 +0200 Subject: [PATCH] start using memcached --- app/scodoc/sco_cache.py | 105 ++++++++++++++++++++++++++++++++++++++-- app/scodoc/sco_core.py | 23 +++++---- app/views/notes.py | 40 --------------- 3 files changed, 115 insertions(+), 53 deletions(-) diff --git a/app/scodoc/sco_cache.py b/app/scodoc/sco_cache.py index f81e5327c..44ed3c409 100644 --- a/app/scodoc/sco_cache.py +++ b/app/scodoc/sco_cache.py @@ -25,14 +25,111 @@ # ############################################################################## -"""Cache simple par etudiant +"""Gestion des caches + Ré-écrite pour ScoDoc8, utilise flask_caching et memcached + + ScoDoc est maintenant multiprocessus / mono-thread, avec un cache en mémoire partagé. """ -import time -# Cache data -class simpleCache(object): +# API ScoDoc8 pour les caches: +# sco_core.get_notes_cache(context).get_NotesTable(context, formsemestre_id) +# => sco_cache.NotesTableCache.get(formsemestre_id) +# +# sco_core.inval_cache(context, formsemestre_id=None, pdfonly=False, formsemestre_id_list=None) +# => deprecated, sco_cache.inval_cache(formsemestre_id=None, pdfonly=False, formsemestre_id_list=None) +# +# +# Nouvelles fonctions: +# sco_cache.NotesTableCache.delete(formsemestre_id) +# sco_cache.NotesTableCache.delete_many(formsemestre_id_list) +# +# Bulletins PDF: +# sco_cache.PDFBulCache.get(formsemestre_id, version) +# sco_cache.PDFBulCache.set(formsemestre_id, version, filename, pdfdoc) +# sco_cache.PDFBulCache.delete(formsemestre_id) suppr. toutes les versions + +# Evaluations: +# sco_cache.EvaluationCache.get(evaluation_id), set(evaluation_id, value), delete(evaluation_id), +# + +from collections import defaultdict + +from flask import g +from flask_caching import Cache + +from app.scodoc.notes_log import log + +CACHE = Cache(config={"CACHE_TYPE": "MemcachedCache"}) # XXX TODO: configuration file + + +class ScoDocCache: + """Cache for ScoDoc objects. + keys are prefixed by the current departement. + """ + + prefix = "" + + @classmethod + def _get_key(cls, oid): + return cls.prefix + g.scodoc_dept + oid + + @classmethod + def get(cls, oid): + """Returns cached evaluation, or None""" + return CACHE.get(cls._get_key(oid)) + + @classmethod + def set(cls, oid, value): + """Store evaluation""" + return CACHE.set(cls._get_key(oid), value) + + @classmethod + def delete(cls, oid): + """Remove from cache""" + CACHE.delete(cls._get_key(oid)) + + @classmethod + def delete_many(cls, oids): + """Remove multiple keys at once""" + CACHE.delete_many(oids) + + +class EvaluationCache(ScoDocCache): + "Cache for evaluations" + prefix = "EVAL" + + +class NotesTableCache(ScoDocCache): + prefix = "NT" + listeners = defaultdict(list) # oid : list of callback functions + + @classmethod + def add_listener(cls, func, oid): + """Add a function which will be called each time and object is modified: + the function will be called as func(oid) + """ + cls.listeners[oid].append(func) + + @classmethod + def delete(cls, oid): + for listener in cls.listeners[oid]: + listener(oid) + super().delete(oid) + + @classmethod + def delete_many(cls, oids): + for oid in oids: + for listener in cls.listeners[oid]: + listener(oid) + super().delete_many(oids) + + +# XXX OBSOLETE A ENLEVER XXX +class simpleCache: + """A simple cache wich cache data for a most "duration" seconds.""" + def __init__(self): self.cache = {} self.inval_cache() # > diff --git a/app/scodoc/sco_core.py b/app/scodoc/sco_core.py index 980aafa29..13a6c3da8 100644 --- a/app/scodoc/sco_core.py +++ b/app/scodoc/sco_core.py @@ -1,12 +1,18 @@ # -*- mode: python -*- # -*- coding: utf-8 -*- -"""essai: ceci serait un module scodoc/sco_xxx.py +"""Gestion des caches + XXX à déplacer dans sco_cache ? + + Ré-écriture en cours pour ScoDoc8, utilise flask_caching et memcached + + ScoDoc est maintenant multiprocessus / mono-thread, avec un cache en mémoire partagé. """ import time import six.moves._thread + from scodoc_manager import sco_mgr import app.scodoc.sco_utils as scu from app.scodoc.notes_log import log @@ -20,6 +26,7 @@ from app.scodoc import sco_cache # NOTES_CACHE_INST = {} # { db_cnx_string : CacheNotesTable instance } +# XXX OBSOLETE... XXX CACHE_formsemestre_inscription = {} CACHE_evaluations = {} @@ -47,13 +54,13 @@ class CacheNotesTable(object): # Cache des NotesTables self.cache = {} # { formsemestre_id : NoteTable instance } # Cache des classeur PDF (bulletins) - self.pdfcache = {} # { formsemestre_id : (filename, pdfdoc) } + self.pdfcache = {} # { (formsemestre_id, version) : (filename, pdfdoc) } # Listeners: self.listeners = scu.DictDefault( defaultvalue={} ) # {formsemestre_id : {listener_id : callback }} - def acquire(self): + def acquire(self): # OBSOLETE "If this thread does not own the cache, acquire the lock" if six.moves._thread.get_ident() != self.owner_thread: if self.lock.locked(): @@ -67,7 +74,7 @@ class CacheNotesTable(object): self.nref += 1 # log('nref=%d' % self.nref) - def release(self): + def release(self): # OBSOLETE "Release the lock" cur_owner_thread = self.owner_thread # log('release: ident=%s (nref=%d)' % (thread.get_ident(), self.nref)) @@ -105,7 +112,7 @@ class CacheNotesTable(object): finally: self.release() - def get_cached_formsemestre_ids(self): + def get_cached_formsemestre_ids(self): # UNUSED "List of currently cached formsemestre_id" return list(self.cache.keys()) @@ -119,8 +126,6 @@ class CacheNotesTable(object): ) try: self.acquire() - if not hasattr(self, "pdfcache"): - self.pdfcache = {} # fix for old zope instances... if formsemestre_id is None: # clear all caches log("----- inval_cache: clearing all caches -----") @@ -206,7 +211,7 @@ class CacheNotesTable(object): """Add a "listener": a function called each time a formsemestre is modified""" self.listeners[formsemestre_id][listener_id] = callback - def remove_listener(self, formsemestre_id, listener_id): + def remove_listener(self, formsemestre_id, listener_id): # UNUSED """Remove a listener. May raise exception if does not exists. """ @@ -250,7 +255,7 @@ def inval_cache( # Cache inscriptions semestres -def get_formsemestre_inscription_cache(context, format=None): +def get_formsemestre_inscription_cache(context): u = sco_mgr.get_db_uri() if u in CACHE_formsemestre_inscription: return CACHE_formsemestre_inscription[u] diff --git a/app/views/notes.py b/app/views/notes.py index 38095a1ad..5584cc568 100644 --- a/app/views/notes.py +++ b/app/views/notes.py @@ -168,46 +168,6 @@ def essai2(context): sco_publish("/essai2", essai2, Permission.ScoImplement) -# --------------------- - - -# --------------- - - -@bp.route("/clearcache") -@permission_required(Permission.ScoView) -@scodoc7func(context) -def clearcache(context, REQUEST=None): - "Efface les caches de notes (utile pendant developpement slt)" - log("*** clearcache request") - # Debugging code: compare results before and after cache reconstruction - # (_should_ be identicals !) - # Compare XML representation - cache = sco_core.get_notes_cache(context) - formsemestre_ids = cache.get_cached_formsemestre_ids() - docs_before = [] - for formsemestre_id in formsemestre_ids: - docs_before.append( - sco_recapcomplet.do_formsemestre_recapcomplet( - context, REQUEST, formsemestre_id, format="xml", xml_nodate=True - ) - ) - # - cache.inval_cache(context) # > - # Rebuild cache (useful only to debug) - docs_after = [] - for formsemestre_id in formsemestre_ids: - docs_after.append( - sco_recapcomplet.do_formsemestre_recapcomplet( - context, REQUEST, formsemestre_id, format="xml", xml_nodate=True - ) - ) - if docs_before != docs_after: - log("clearcache: inconsistency !") - txt = "before=" + repr(docs_before) + "\n\nafter=" + repr(docs_after) + "\n" - log(txt) - sendAlarm(context, "clearcache: inconsistency !", txt) - # -------------------------------------------------------------------- #