# -*- mode: python -*-
# -*- coding: utf-8 -*-

##############################################################################
#
# Gestion scolarite IUT
#
# Copyright (c) 1999 - 2021 Emmanuel Viennet.  All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#   Emmanuel Viennet      emmanuel.viennet@viennet.net
#
##############################################################################

""" Gestion des absences (v4)

C'est la partie la plus ancienne de ScoDoc, et elle est à revoir.

L'API de plus bas niveau est en gros:

 AnnuleAbsencesDatesNoJust(etudid, dates)
 CountAbs(etudid, debut, fin, matin=None, moduleimpl_id=None)
 CountAbsJust(etudid, debut, fin, matin=None, moduleimpl_id=None)
 ListeAbsJust(etudid, datedebut)  [pas de fin ?]
 ListeAbsNonJust(etudid, datedebut)  [pas de fin ?]
 ListeJustifs(etudid, datedebut, datefin=None, only_no_abs=True)

 ListeAbsJour(date, am=True, pm=True, is_abs=None, is_just=None)
 ListeAbsNonJustJour(date, am=True, pm=True)


"""

import string
import re
import time
import datetime
import dateutil
import dateutil.parser
import calendar
import urllib
import cgi
import jaxml

# ---------------
from sco_zope import *  # pylint: disable=unused-wildcard-import

# ---------------
import sco_utils as scu
import notesdb
from notes_log import log
from scolog import logdb
from sco_permissions import ScoAbsAddBillet, ScoAbsChange, ScoView
from sco_exceptions import ScoValueError, ScoInvalidDateError
from TrivialFormulator import TrivialFormulator, TF
from gen_tables import GenTable
import html_sco_header
import scolars
import sco_formsemestre
import sco_moduleimpl
import sco_groups
import sco_groups_view
import sco_excel
import sco_abs_notification, sco_abs_views
import sco_compute_moy
import sco_abs
from sco_abs import ddmmyyyy

CSSSTYLES = html_sco_header.BOOTSTRAP_MULTISELECT_CSS


def _toboolean(x):
    "convert a value to boolean (ensure backward compat with OLD intranet code)"
    if type(x) == type(""):
        x = x.lower()
    if x and x != "false":  # backward compat...
        return True
    else:
        return False


class ZAbsences(
    ObjectManager, PropertyManager, RoleManager, Item, Persistent, Implicit
):

    "ZAbsences object"

    meta_type = "ZAbsences"
    security = ClassSecurityInfo()

    # This is the list of the methods associated to 'tabs' in the ZMI
    # Be aware that The first in the list is the one shown by default, so if
    # the 'View' tab is the first, you will never see your tabs by cliquing
    # on the object.
    manage_options = (
        ({"label": "Contents", "action": "manage_main"},)
        + PropertyManager.manage_options  # add the 'Properties' tab
        + ({"label": "View", "action": "index_html"},)
        + Item.manage_options  # add the 'Undo' & 'Owner' tab
        + RoleManager.manage_options  # add the 'Security' tab
    )

    # no permissions, only called from python
    def __init__(self, id, title):
        "initialise a new instance"
        self.id = id
        self.title = title

    # The form used to edit this object
    # def manage_editZAbsences(self, title, RESPONSE=None):
    #     "Changes the instance values"
    #     self.title = title
    #     self._p_changed = 1
    #     RESPONSE.redirect("manage_editForm")

    # --------------------------------------------------------------------
    #
    #   ABSENCES (top level)
    #
    # --------------------------------------------------------------------
    # used to view content of the object
    security.declareProtected(ScoView, "index_html")
    index_html = sco_abs_views.absences_index_html

    security.declareProtected(ScoView, "EtatAbsences")
    EtatAbsences = sco_abs_views.EtatAbsences

    security.declareProtected(ScoView, "CalAbs")
    CalAbs = sco_abs_views.CalAbs

    security.declareProtected(ScoAbsChange, "SignaleAbsenceEtud")
    SignaleAbsenceEtud = sco_abs_views.SignaleAbsenceEtud
    security.declareProtected(ScoAbsChange, "doSignaleAbsence")
    doSignaleAbsence = sco_abs_views.doSignaleAbsence

    security.declareProtected(ScoAbsChange, "JustifAbsenceEtud")
    JustifAbsenceEtud = sco_abs_views.JustifAbsenceEtud
    security.declareProtected(ScoAbsChange, "doJustifAbsence")
    doJustifAbsence = sco_abs_views.doJustifAbsence

    security.declareProtected(ScoAbsChange, "AnnuleAbsenceEtud")
    AnnuleAbsenceEtud = sco_abs_views.AnnuleAbsenceEtud
    security.declareProtected(ScoAbsChange, "doAnnuleAbsence")
    doAnnuleAbsence = sco_abs_views.doAnnuleAbsence
    security.declareProtected(ScoAbsChange, "doAnnuleJustif")
    doAnnuleJustif = sco_abs_views.doAnnuleJustif

    security.declareProtected(ScoView, "ListeAbsEtud")
    ListeAbsEtud = sco_abs_views.ListeAbsEtud

    # --------------------------------------------------------------------
    #
    #   SQL METHODS
    #
    # --------------------------------------------------------------------

    def _AddAbsence(
        self,
        etudid,
        jour,
        matin,
        estjust,
        REQUEST,
        description=None,
        moduleimpl_id=None,
    ):
        "Ajoute une absence dans la bd"
        # unpublished
        if self._isFarFutur(jour):
            raise ScoValueError("date absence trop loin dans le futur !")
        estjust = _toboolean(estjust)
        matin = _toboolean(matin)
        cnx = self.GetDBConnexion()
        cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
        cursor.execute(
            "insert into absences (etudid,jour,estabs,estjust,matin,description, moduleimpl_id) values (%(etudid)s, %(jour)s, TRUE, %(estjust)s, %(matin)s, %(description)s, %(moduleimpl_id)s )",
            vars(),
        )
        logdb(
            REQUEST,
            cnx,
            "AddAbsence",
            etudid=etudid,
            msg="JOUR=%(jour)s,MATIN=%(matin)s,ESTJUST=%(estjust)s,description=%(description)s,moduleimpl_id=%(moduleimpl_id)s"
            % vars(),
        )
        cnx.commit()
        sco_abs.invalidateAbsEtudDate(self, etudid, jour)
        sco_abs_notification.abs_notify(self, etudid, jour)

    def _AddJustif(self, etudid, jour, matin, REQUEST, description=None):
        "Ajoute un justificatif dans la base"
        # unpublished
        if self._isFarFutur(jour):
            raise ScoValueError("date justificatif trop loin dans le futur !")
        matin = _toboolean(matin)
        cnx = self.GetDBConnexion()
        cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
        cursor.execute(
            "insert into absences (etudid,jour,estabs,estjust,matin, description) values (%(etudid)s,%(jour)s, FALSE, TRUE, %(matin)s, %(description)s )",
            vars(),
        )
        logdb(
            REQUEST,
            cnx,
            "AddJustif",
            etudid=etudid,
            msg="JOUR=%(jour)s,MATIN=%(matin)s" % vars(),
        )
        cnx.commit()
        sco_abs.invalidateAbsEtudDate(self, etudid, jour)

    def _AnnuleAbsence(self, etudid, jour, matin, moduleimpl_id=None, REQUEST=None):
        """Annule une absence ds base
        Si moduleimpl_id, n'annule que pour ce module
        """
        # unpublished
        matin = _toboolean(matin)
        cnx = self.GetDBConnexion()
        cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
        req = "delete from absences where jour=%(jour)s and matin=%(matin)s and etudid=%(etudid)s and estabs"
        if moduleimpl_id:
            req += " and moduleimpl_id=%(moduleimpl_id)s"
        cursor.execute(req, vars())
        logdb(
            REQUEST,
            cnx,
            "AnnuleAbsence",
            etudid=etudid,
            msg="JOUR=%(jour)s,MATIN=%(matin)s,moduleimpl_id=%(moduleimpl_id)s"
            % vars(),
        )
        cnx.commit()
        sco_abs.invalidateAbsEtudDate(self, etudid, jour)

    def _AnnuleJustif(self, etudid, jour, matin, REQUEST=None):
        "Annule un justificatif"
        # unpublished
        matin = _toboolean(matin)
        cnx = self.GetDBConnexion()
        cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
        cursor.execute(
            "delete from absences where jour=%(jour)s and matin=%(matin)s and etudid=%(etudid)s and ESTJUST AND NOT ESTABS",
            vars(),
        )
        cursor.execute(
            "update absences set estjust=false where jour=%(jour)s and matin=%(matin)s and etudid=%(etudid)s",
            vars(),
        )
        logdb(
            REQUEST,
            cnx,
            "AnnuleJustif",
            etudid=etudid,
            msg="JOUR=%(jour)s,MATIN=%(matin)s" % vars(),
        )
        cnx.commit()
        sco_abs.invalidateAbsEtudDate(self, etudid, jour)

    # Fonction inutile à supprimer (gestion moduleimpl_id incorrecte):
    # def _AnnuleAbsencesPeriodNoJust(self, etudid, datedebut, datefin,
    #                                 moduleimpl_id=None, REQUEST=None):
    #     """Supprime les absences entre ces dates (incluses).
    #     mais ne supprime pas les justificatifs.
    #     """
    #     # unpublished
    #     cnx = self.GetDBConnexion()
    #     cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
    #     # supr les absences non justifiees
    #     cursor.execute("delete from absences where etudid=%(etudid)s and (not estjust) and moduleimpl_id=(moduleimpl_id)s and jour BETWEEN %(datedebut)s AND %(datefin)s",
    #                    vars() )
    #     # s'assure que les justificatifs ne sont pas "absents"
    #     cursor.execute("update absences set estabs=FALSE where  etudid=%(etudid)s and jour and moduleimpl_id=(moduleimpl_id)s  BETWEEN %(datedebut)s AND %(datefin)s", vars())
    #     logdb(REQUEST, cnx, 'AnnuleAbsencesPeriodNoJust', etudid=etudid,
    #           msg='%(datedebut)s - %(datefin)s - (moduleimpl_id)s'%vars())
    #     cnx.commit()
    #     sco_abs.invalidateAbsEtudDate(self, etudid, datedebut)
    #     sco_abs.invalidateAbsEtudDate(self, etudid, datefin) # si un semestre commence apres datedebut et termine avant datefin, il ne sera pas invalide. Tant pis ;-)

    security.declareProtected(ScoAbsChange, "AnnuleAbsencesDatesNoJust")

    def AnnuleAbsencesDatesNoJust(
        self, etudid, dates, moduleimpl_id=None, REQUEST=None
    ):
        """Supprime les absences aux dates indiquées
        mais ne supprime pas les justificatifs.
        """
        # log('AnnuleAbsencesDatesNoJust: moduleimpl_id=%s' % moduleimpl_id)
        if not dates:
            return
        date0 = dates[0]
        if len(date0.split(":")) == 2:
            # am/pm is present
            for date in dates:
                jour, ampm = date.split(":")
                if ampm == "am":
                    matin = 1
                elif ampm == "pm":
                    matin = 0
                else:
                    raise ValueError("invalid ampm !")
                self._AnnuleAbsence(etudid, jour, matin, moduleimpl_id, REQUEST)
            return
        cnx = self.GetDBConnexion()
        cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
        # supr les absences non justifiees
        for date in dates:
            cursor.execute(
                "delete from absences where etudid=%(etudid)s and (not estjust) and jour=%(date)s and moduleimpl_id=%(moduleimpl_id)s",
                vars(),
            )
            sco_abs.invalidateAbsEtudDate(self, etudid, date)
        # s'assure que les justificatifs ne sont pas "absents"
        for date in dates:
            cursor.execute(
                "update absences set estabs=FALSE where  etudid=%(etudid)s and jour=%(date)s and moduleimpl_id=%(moduleimpl_id)s",
                vars(),
            )
        if dates:
            date0 = dates[0]
        else:
            date0 = None
        if len(dates) > 1:
            date1 = dates[1]
        else:
            date1 = None
        logdb(
            REQUEST,
            cnx,
            "AnnuleAbsencesDatesNoJust",
            etudid=etudid,
            msg="%s - %s - %s" % (date0, date1, moduleimpl_id),
        )
        cnx.commit()

    def ListAbsInRange(
        self, etudid, debut, fin, matin=None, moduleimpl_id=None, cursor=None
    ):
        """Liste des absences entre deux dates.

        Args:
            etudid
            debut   string iso date ("2020-03-12")
            end     string iso date ("2020-03-12")
            matin   None, True, False
            moduleimpl_id
        """
        if matin != None:
            matin = _toboolean(matin)
            ismatin = " AND A.MATIN = %(matin)s "
        else:
            ismatin = ""
        if moduleimpl_id:
            modul = " AND A.MODULEIMPL_ID = %(moduleimpl_id)s "
        else:
            modul = ""
        if not cursor:
            cnx = self.GetDBConnexion()
            cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
        cursor.execute(
            """
    SELECT DISTINCT A.JOUR, A.MATIN
    FROM ABSENCES A
    WHERE A.ETUDID = %(etudid)s
      AND A.ESTABS"""
            + ismatin
            + modul
            + """
      AND A.JOUR BETWEEN %(debut)s AND %(fin)s
          """,
            vars(),
        )
        res = cursor.dictfetchall()
        return res

    security.declareProtected(ScoView, "CountAbs")

    def CountAbs(self, etudid, debut, fin, matin=None, moduleimpl_id=None):
        """CountAbs
        matin= 1 ou 0.

        Returns:
            An integer.
        """
        return len(
            self.ListAbsInRange(
                etudid, debut, fin, matin=matin, moduleimpl_id=moduleimpl_id
            )
        )

    security.declareProtected(ScoView, "CountAbsJust")

    def CountAbsJust(self, etudid, debut, fin, matin=None, moduleimpl_id=None):
        "Count just. abs"
        if matin != None:
            matin = _toboolean(matin)
            ismatin = " AND A.MATIN = %(matin)s "
        else:
            ismatin = ""
        if moduleimpl_id:
            modul = " AND A.MODULEIMPL_ID = %(moduleimpl_id)s "
        else:
            modul = ""
        cnx = self.GetDBConnexion()
        cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
        cursor.execute(
            """SELECT COUNT(*) AS NbAbsJust FROM (
  SELECT DISTINCT A.JOUR, A.MATIN
  FROM ABSENCES A, ABSENCES B
  WHERE A.ETUDID = %(etudid)s
      AND A.ETUDID = B.ETUDID 
      AND A.JOUR = B.JOUR AND A.MATIN = B.MATIN
      AND A.JOUR BETWEEN %(debut)s AND %(fin)s
      AND A.ESTABS AND (A.ESTJUST OR B.ESTJUST)"""
            + ismatin
            + modul
            + """
) AS tmp
        """,
            vars(),
        )
        res = cursor.fetchone()[0]
        return res

    def _ListeAbsDate(self, etudid, beg_date, end_date):
        # Liste des absences et justifs entre deux dates
        cnx = self.GetDBConnexion()
        cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
        cursor.execute(
            """SELECT jour, matin, estabs, estjust, description FROM ABSENCES A 
     WHERE A.ETUDID = %(etudid)s
     AND A.jour >= %(beg_date)s 
     AND A.jour <= %(end_date)s 
         """,
            vars(),
        )
        Abs = cursor.dictfetchall()
        # remove duplicates
        A = {}  # { (jour, matin) : abs }
        for a in Abs:
            jour, matin = a["jour"], a["matin"]
            if (jour, matin) in A:
                # garde toujours la description
                a["description"] = a["description"] or A[(jour, matin)]["description"]
                # et la justif:
                a["estjust"] = a["estjust"] or A[(jour, matin)]["estjust"]
                a["estabs"] = a["estabs"] or A[(jour, matin)]["estabs"]
                A[(jour, matin)] = a
            else:
                A[(jour, matin)] = a
            if A[(jour, matin)]["description"] is None:
                A[(jour, matin)]["description"] = ""
            # add hours: matin = 8:00 - 12:00, apresmidi = 12:00 - 18:00
            dat = "%04d-%02d-%02d" % (a["jour"].year, a["jour"].month, a["jour"].day)
            if a["matin"]:
                A[(jour, matin)]["begin"] = dat + " 08:00:00"
                A[(jour, matin)]["end"] = dat + " 11:59:59"
            else:
                A[(jour, matin)]["begin"] = dat + " 12:00:00"
                A[(jour, matin)]["end"] = dat + " 17:59:59"
        # sort
        R = A.values()
        R.sort(key=lambda x: (x["begin"]))
        return R

    security.declareProtected(ScoView, "ListeAbsJust")

    def ListeAbsJust(self, etudid, datedebut):
        "Liste des absences justifiees (par ordre chronologique)"
        cnx = self.GetDBConnexion()
        cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
        cursor.execute(
            """SELECT DISTINCT A.ETUDID, A.JOUR, A.MATIN FROM ABSENCES A, ABSENCES B
 WHERE A.ETUDID = %(etudid)s
 AND A.ETUDID = B.ETUDID 
 AND A.JOUR = B.JOUR AND A.MATIN = B.MATIN AND A.JOUR >= %(datedebut)s
 AND A.ESTABS AND (A.ESTJUST OR B.ESTJUST)
 ORDER BY A.JOUR
        """,
            vars(),
        )
        A = cursor.dictfetchall()
        for a in A:
            a["description"] = self._GetAbsDescription(a, cursor=cursor)
        return A

    security.declareProtected(ScoView, "ListeAbsNonJust")

    def ListeAbsNonJust(self, etudid, datedebut):
        "Liste des absences NON justifiees (par ordre chronologique)"
        cnx = self.GetDBConnexion()
        cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
        cursor.execute(
            """SELECT ETUDID, JOUR, MATIN FROM ABSENCES A 
    WHERE A.ETUDID = %(etudid)s
    AND A.estabs 
    AND A.jour >= %(datedebut)s
    EXCEPT SELECT ETUDID, JOUR, MATIN FROM ABSENCES B 
    WHERE B.estjust 
    AND B.ETUDID = %(etudid)s
    ORDER BY JOUR
        """,
            vars(),
        )
        A = cursor.dictfetchall()
        for a in A:
            a["description"] = self._GetAbsDescription(a, cursor=cursor)
        return A

    security.declareProtected(ScoView, "ListeJustifs")

    def ListeJustifs(self, etudid, datedebut, datefin=None, only_no_abs=False):
        """Liste des justificatifs (sans absence relevée) à partir d'une date,
        ou, si datefin spécifié, entre deux dates.
        Si only_no_abs: seulement les justificatifs correspondant aux jours sans absences relevées.
        """
        cnx = self.GetDBConnexion()
        cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
        req = """SELECT DISTINCT ETUDID, JOUR, MATIN FROM ABSENCES A
 WHERE A.ETUDID = %(etudid)s
 AND A.ESTJUST
 AND A.JOUR >= %(datedebut)s"""
        if datefin:
            req += """AND A.JOUR <= %(datefin)s"""
        if only_no_abs:
            req += """
 EXCEPT SELECT ETUDID, JOUR, MATIN FROM ABSENCES B 
 WHERE B.estabs
 AND B.ETUDID = %(etudid)s
        """
        cursor.execute(req, vars())
        A = cursor.dictfetchall()
        for a in A:
            a["description"] = self._GetAbsDescription(a, cursor=cursor)

        return A

    def _GetAbsDescription(self, a, cursor=None):
        "Description associee a l'absence"
        if not cursor:
            cnx = self.GetDBConnexion()
            cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
        a = a.copy()
        # a['jour'] = a['jour'].date()
        if a["matin"]:  # devrait etre booleen... :-(
            a["matin"] = True
        else:
            a["matin"] = False
        cursor.execute(
            """select * from absences where etudid=%(etudid)s and jour=%(jour)s and matin=%(matin)s order by entry_date desc""",
            a,
        )
        A = cursor.dictfetchall()
        desc = None
        module = ""
        for a in A:
            if a["description"]:
                desc = a["description"]
            if a["moduleimpl_id"] and a["moduleimpl_id"] != "NULL":
                # Trouver le nom du module
                Mlist = sco_moduleimpl.do_moduleimpl_withmodule_list(
                    self.Notes, moduleimpl_id=a["moduleimpl_id"]
                )
                if Mlist:
                    M = Mlist[0]
                    module += "%s " % M["module"]["code"]

        if desc:
            return "(%s) %s" % (desc, module)
        if module:
            return module
        return ""

    security.declareProtected(ScoView, "ListeAbsJour")

    def ListeAbsJour(self, date, am=True, pm=True, is_abs=True, is_just=None):
        """Liste des absences et/ou justificatifs ce jour.
        is_abs: None (peu importe), True, False
        is_just: idem
        """
        cnx = self.GetDBConnexion()
        cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
        req = """SELECT DISTINCT etudid, jour, matin FROM ABSENCES A 
    WHERE A.jour = %(date)s
    """
        if is_abs != None:
            req += " AND A.estabs = %(is_abs)s"
        if is_just != None:
            req += " AND A.estjust = %(is_just)s"
        if not am:
            req += " AND NOT matin "
        if not pm:
            req += " AND matin"

        cursor.execute(req, {"date": date, "is_just": is_just, "is_abs": is_abs})
        A = cursor.dictfetchall()
        for a in A:
            a["description"] = self._GetAbsDescription(a, cursor=cursor)
        return A

    security.declareProtected(ScoView, "ListeAbsNonJustJour")

    def ListeAbsNonJustJour(self, date, am=True, pm=True):
        "Liste des absences non justifiees ce jour"
        cnx = self.GetDBConnexion()
        cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
        reqa = ""
        if not am:
            reqa += " AND NOT matin "
        if not pm:
            reqa += " AND matin "
        req = (
            """SELECT  etudid, jour, matin FROM ABSENCES A 
    WHERE A.estabs 
    AND A.jour = %(date)s
    """
            + reqa
            + """EXCEPT SELECT etudid, jour, matin FROM ABSENCES B 
    WHERE B.estjust AND B.jour = %(date)s"""
            + reqa
        )

        cursor.execute(req, {"date": date})
        A = cursor.dictfetchall()
        for a in A:
            a["description"] = self._GetAbsDescription(a, cursor=cursor)
        return A

    security.declareProtected(ScoAbsChange, "doSignaleAbsenceGrSemestre")

    def doSignaleAbsenceGrSemestre(
        self,
        moduleimpl_id=None,
        abslist=[],
        dates="",
        etudids="",
        destination=None,
        REQUEST=None,
    ):
        """Enregistre absences aux dates indiquees (abslist et dates).
        dates est une liste de dates ISO (séparées par des ',').
        Efface les absences aux dates indiquées par dates,
        ou bien ajoute celles de abslist.
        """
        if etudids:
            etudids = etudids.split(",")
        else:
            etudids = []
        if dates:
            dates = dates.split(",")
        else:
            dates = []

        # 1- Efface les absences
        if dates:
            for etudid in etudids:
                self.AnnuleAbsencesDatesNoJust(etudid, dates, moduleimpl_id, REQUEST)
            return "Absences effacées"

        # 2- Ajoute les absences
        if abslist:
            self._add_abslist(abslist, REQUEST, moduleimpl_id)
            return "Absences ajoutées"

        return ""

    def _add_abslist(self, abslist, REQUEST, moduleimpl_id=None):
        for a in abslist:
            etudid, jour, ampm = a.split(":")
            if ampm == "am":
                matin = 1
            elif ampm == "pm":
                matin = 0
            else:
                raise ValueError("invalid ampm !")
            # ajoute abs si pas deja absent
            if self.CountAbs(etudid, jour, jour, matin, moduleimpl_id) == 0:
                self._AddAbsence(etudid, jour, matin, 0, REQUEST, "", moduleimpl_id)

    # --- Misc tools.... ------------------

    def _isFarFutur(self, jour):
        # check si jour est dans le futur "lointain"
        # pour autoriser les saisies dans le futur mais pas a plus de 6 mois
        y, m, d = [int(x) for x in jour.split("-")]
        j = datetime.date(y, m, d)
        # 6 mois ~ 182 jours:
        return j - datetime.date.today() > datetime.timedelta(182)

    # ------------ HTML Interfaces
    security.declareProtected(ScoAbsChange, "SignaleAbsenceGrHebdo")

    def SignaleAbsenceGrHebdo(
        self, datelundi, group_ids=[], destination="", moduleimpl_id=None, REQUEST=None
    ):
        "Saisie hebdomadaire des absences"
        if not moduleimpl_id:
            moduleimpl_id = None

        groups_infos = sco_groups_view.DisplayedGroupsInfos(
            self, group_ids, moduleimpl_id=moduleimpl_id, REQUEST=REQUEST
        )
        if not groups_infos.members:
            return (
                self.sco_header(page_title="Saisie des absences", REQUEST=REQUEST)
                + "<h3>Aucun étudiant !</h3>"
                + self.sco_footer(REQUEST)
            )

        base_url = "SignaleAbsenceGrHebdo?datelundi=%s&amp;%s&amp;destination=%s" % (
            datelundi,
            groups_infos.groups_query_args,
            urllib.quote(destination),
        )

        formsemestre_id = groups_infos.formsemestre_id
        require_module = self.get_preference("abs_require_module", formsemestre_id)
        etuds = [
            self.getEtudInfo(etudid=m["etudid"], filled=True)[0]
            for m in groups_infos.members
        ]
        # Restreint aux inscrits au module sélectionné
        if moduleimpl_id:
            mod_inscrits = set(
                [
                    x["etudid"]
                    for x in sco_moduleimpl.do_moduleimpl_inscription_list(
                        self.Notes, moduleimpl_id=moduleimpl_id
                    )
                ]
            )
            etuds_inscrits_module = [e for e in etuds if e["etudid"] in mod_inscrits]
            if etuds_inscrits_module:
                etuds = etuds_inscrits_module
            else:
                # Si aucun etudiant n'est inscrit au module choisi...
                moduleimpl_id = None
        nt = self.Notes._getNotesCache().get_NotesTable(self.Notes, formsemestre_id)
        sem = sco_formsemestre.do_formsemestre_list(
            self, {"formsemestre_id": formsemestre_id}
        )[0]

        # calcule dates jours de cette semaine
        # liste de dates iso "yyyy-mm-dd"
        datessem = [notesdb.DateDMYtoISO(datelundi)]
        for _ in sco_abs.day_names(self)[1:]:
            datessem.append(sco_abs.next_iso_day(self, datessem[-1]))
        #
        if groups_infos.tous_les_etuds_du_sem:
            gr_tit = "en"
        else:
            if len(groups_infos.group_ids) > 1:
                p = "des groupes"
            else:
                p = "du groupe"
            gr_tit = (
                p + ' <span class="fontred">' + groups_infos.groups_titles + "</span>"
            )

        H = [
            self.sco_header(
                page_title="Saisie hebdomadaire des absences",
                init_qtip=True,
                javascripts=html_sco_header.BOOTSTRAP_MULTISELECT_JS
                + [
                    "js/etud_info.js",
                    "js/abs_ajax.js",
                    "js/groups_view.js",
                ],
                cssstyles=CSSSTYLES,
                no_side_bar=1,
                REQUEST=REQUEST,
            ),
            """<table border="0" cellspacing="16"><tr><td>
            <h2>Saisie des absences %s %s, 
            <span class="fontred">semaine du lundi %s</span></h2>
            <div>
            <form id="group_selector" method="get">
            <input type="hidden" name="formsemestre_id" id="formsemestre_id" value="%s"/>
            <input type="hidden" name="datelundi" id="datelundi" value="%s"/>
            <input type="hidden" name="destination" id="destination" value="%s"/>
            <input type="hidden" name="moduleimpl_id" id="moduleimpl_id_o" value="%s"/>
            Groupes: %s
            </form>
            <form id="abs_form">
            """
            % (
                gr_tit,
                sem["titre_num"],
                datelundi,
                groups_infos.formsemestre_id,
                datelundi,
                destination,
                moduleimpl_id or "",
                sco_groups_view.menu_groups_choice(
                    self, groups_infos, submit_on_change=True
                ),
            ),
        ]
        #
        modimpls_list = []
        # Initialize with first student
        ues = nt.get_ues(etudid=etuds[0]["etudid"])
        for ue in ues:
            modimpls_list += nt.get_modimpls(ue_id=ue["ue_id"])

        # Add modules other students are subscribed to
        for etud in etuds[1:]:
            modimpls_etud = []
            ues = nt.get_ues(etudid=etud["etudid"])
            for ue in ues:
                modimpls_etud += nt.get_modimpls(ue_id=ue["ue_id"])
            modimpls_list += [m for m in modimpls_etud if m not in modimpls_list]

        menu_module = ""
        for modimpl in modimpls_list:
            if modimpl["moduleimpl_id"] == moduleimpl_id:
                sel = "selected"
            else:
                sel = ""
            menu_module += (
                """<option value="%(modimpl_id)s" %(sel)s>%(modname)s</option>\n"""
                % {
                    "modimpl_id": modimpl["moduleimpl_id"],
                    "modname": modimpl["module"]["code"]
                    + " "
                    + (modimpl["module"]["abbrev"] or modimpl["module"]["titre"]),
                    "sel": sel,
                }
            )
        if moduleimpl_id:
            sel = ""
        else:
            sel = "selected"  # aucun module specifie

        H.append(
            """Module concerné: 
            <select id="moduleimpl_id" name="moduleimpl_id" onchange="change_moduleimpl('%(url)s')">
            <option value="" %(sel)s>non spécifié</option>
            %(menu_module)s
            </select>
            </div>"""
            % {"menu_module": menu_module, "url": base_url, "sel": sel}
        )

        H += self._gen_form_saisie_groupe(
            etuds, datessem, destination, moduleimpl_id, require_module
        )

        H.append(self.sco_footer(REQUEST))
        return "\n".join(H)

    security.declareProtected(ScoAbsChange, "SignaleAbsenceGrSemestre")

    def SignaleAbsenceGrSemestre(
        self,
        datedebut,
        datefin,
        destination="",
        group_ids=[],  # list of groups to display
        nbweeks=4,  # ne montre que les nbweeks dernieres semaines
        moduleimpl_id=None,
        REQUEST=None,
    ):
        """Saisie des absences sur une journée sur un semestre (ou intervalle de dates) entier"""
        groups_infos = sco_groups_view.DisplayedGroupsInfos(
            self, group_ids, REQUEST=REQUEST
        )
        if not groups_infos.members:
            return (
                self.sco_header(page_title="Saisie des absences", REQUEST=REQUEST)
                + "<h3>Aucun étudiant !</h3>"
                + self.sco_footer(REQUEST)
            )
        formsemestre_id = groups_infos.formsemestre_id
        require_module = self.get_preference("abs_require_module", formsemestre_id)
        etuds = [
            self.getEtudInfo(etudid=m["etudid"], filled=True)[0]
            for m in groups_infos.members
        ]
        # Restreint aux inscrits au module sélectionné
        if moduleimpl_id:
            mod_inscrits = set(
                [
                    x["etudid"]
                    for x in sco_moduleimpl.do_moduleimpl_inscription_list(
                        self.Notes, moduleimpl_id=moduleimpl_id
                    )
                ]
            )
            etuds = [e for e in etuds if e["etudid"] in mod_inscrits]
        if not moduleimpl_id:
            moduleimpl_id = None
        base_url_noweeks = "SignaleAbsenceGrSemestre?datedebut=%s&amp;datefin=%s&amp;%s&amp;destination=%s" % (
            datedebut,
            datefin,
            groups_infos.groups_query_args,
            urllib.quote(destination),
        )
        base_url = (
            base_url_noweeks + "&amp;nbweeks=%s" % nbweeks
        )  # sans le moduleimpl_id

        if etuds:
            nt = self.Notes._getNotesCache().get_NotesTable(self.Notes, formsemestre_id)
            sem = sco_formsemestre.do_formsemestre_list(
                self, {"formsemestre_id": formsemestre_id}
            )[0]
        work_saturday = sco_abs.is_work_saturday(self)
        jourdebut = ddmmyyyy(datedebut, work_saturday=work_saturday)
        jourfin = ddmmyyyy(datefin, work_saturday=work_saturday)
        today = ddmmyyyy(
            time.strftime("%d/%m/%Y", time.localtime()),
            work_saturday=work_saturday,
        )
        today.next()
        if jourfin > today:  # ne propose jamais les semaines dans le futur
            jourfin = today
        if jourdebut > today:
            raise ScoValueError("date de début dans le futur (%s) !" % jourdebut)
        #
        if not jourdebut.iswork() or jourdebut > jourfin:
            raise ValueError(
                "date debut invalide (%s, ouvrable=%d)"
                % (str(jourdebut), jourdebut.iswork())
            )
        # calcule dates
        dates = []  # ddmmyyyy instances
        d = ddmmyyyy(datedebut, work_saturday=work_saturday)
        while d <= jourfin:
            dates.append(d)
            d = d.next(7)  # avance d'une semaine
        #
        msg = "Montrer seulement les 4 dernières semaines"
        nwl = 4
        if nbweeks:
            nbweeks = int(nbweeks)
            if nbweeks > 0:
                dates = dates[-nbweeks:]
                msg = "Montrer toutes les semaines"
                nwl = 0
        url_link_semaines = base_url_noweeks + "&amp;nbweeks=%s" % nwl
        if moduleimpl_id:
            url_link_semaines += "&amp;moduleimpl_id=" + moduleimpl_id
        #
        dates = [x.ISO() for x in dates]
        dayname = sco_abs.day_names(self)[jourdebut.weekday]

        if groups_infos.tous_les_etuds_du_sem:
            gr_tit = "en"
        else:
            if len(groups_infos.group_ids) > 1:
                p = "des groupes "
            else:
                p = "du groupe "
            gr_tit = (
                p + '<span class="fontred">' + groups_infos.groups_titles + "</span>"
            )

        H = [
            self.sco_header(
                page_title="Saisie des absences",
                init_qtip=True,
                javascripts=["js/etud_info.js", "js/abs_ajax.js"],
                no_side_bar=1,
                REQUEST=REQUEST,
            ),
            """<table border="0" cellspacing="16"><tr><td>
              <h2>Saisie des absences %s %s, 
              les <span class="fontred">%s</span></h2>
              <p>
              <a href="%s">%s</a>
              <form id="abs_form" action="doSignaleAbsenceGrSemestre" method="post">
              """
            % (gr_tit, sem["titre_num"], dayname, url_link_semaines, msg),
        ]
        #
        if etuds:
            modimpls_list = []
            # Initialize with first student
            ues = nt.get_ues(etudid=etuds[0]["etudid"])
            for ue in ues:
                modimpls_list += nt.get_modimpls(ue_id=ue["ue_id"])

            # Add modules other students are subscribed to
            for etud in etuds[1:]:
                modimpls_etud = []
                ues = nt.get_ues(etudid=etud["etudid"])
                for ue in ues:
                    modimpls_etud += nt.get_modimpls(ue_id=ue["ue_id"])
                modimpls_list += [m for m in modimpls_etud if m not in modimpls_list]

            menu_module = ""
            for modimpl in modimpls_list:
                if modimpl["moduleimpl_id"] == moduleimpl_id:
                    sel = "selected"
                else:
                    sel = ""
                menu_module += (
                    """<option value="%(modimpl_id)s" %(sel)s>%(modname)s</option>\n"""
                    % {
                        "modimpl_id": modimpl["moduleimpl_id"],
                        "modname": modimpl["module"]["code"]
                        + " "
                        + (modimpl["module"]["abbrev"] or modimpl["module"]["titre"]),
                        "sel": sel,
                    }
                )
            if moduleimpl_id:
                sel = ""
            else:
                sel = "selected"  # aucun module specifie
            H.append(
                """<p>
    Module concerné par ces absences (%(optionel_txt)s): 
    <select id="moduleimpl_id" name="moduleimpl_id" 
    onchange="document.location='%(url)s&amp;moduleimpl_id='+document.getElementById('moduleimpl_id').value">
    <option value="" %(sel)s>non spécifié</option>
    %(menu_module)s
    </select>
</p>"""
                % {
                    "menu_module": menu_module,
                    "url": base_url,
                    "sel": sel,
                    "optionel_txt": '<span class="redboldtext">requis</span>'
                    if require_module
                    else "optionnel",
                }
            )

        H += self._gen_form_saisie_groupe(
            etuds, dates, destination, moduleimpl_id, require_module
        )
        H.append(self.sco_footer(REQUEST))
        return "\n".join(H)

    def _gen_form_saisie_groupe(
        self, etuds, dates, destination="", moduleimpl_id=None, require_module=False
    ):
        """Formulaire saisie absences

        Args:
            etuds: liste des étudiants
            dates: liste ordonnée de dates iso, par exemple: [ '2020-12-24', ... ]
            moduleimpl_id: optionnel, module concerné.
        """
        H = [
            """
        <script type="text/javascript">
        $(function() {
            $(".abs_form_table input").prop( "disabled", %s );
        });
        function colorize(obj) {
             if (obj.checked) {
                 obj.parentNode.className = 'absent';
             } else {
                 obj.parentNode.className = 'present';
             }
        }
        function on_toggled(obj, etudid, dat) {
            colorize(obj);
            if (obj.checked) {
                ajaxFunction('add', etudid, dat);
            } else {
                ajaxFunction('remove', etudid, dat);
            }
        }
        </script>
        <div id="AjaxDiv"></div>
        <br/>
        <table rules="cols" frame="box" class="abs_form_table">
        <tr><th class="formabs_contetud">%d étudiants</th>
        """
            % (
                "true" if (require_module and not moduleimpl_id) else "false",
                len(etuds),
            )
        ]
        # Dates
        odates = [datetime.date(*[int(x) for x in d.split("-")]) for d in dates]
        begin = dates[0]
        end = dates[-1]
        # Titres colonnes
        noms_jours = []  # eg [ "Lundi", "mardi", "Samedi", ... ]
        jn = sco_abs.day_names(self)
        for d in odates:
            idx_jour = d.weekday()
            noms_jours.append(jn[idx_jour])
        for jour in noms_jours:
            H.append(
                '<th colspan="2" width="100px" style="padding-left: 5px; padding-right: 5px;">'
                + jour
                + "</th>"
            )
        H.append("</tr><tr><td>&nbsp;</td>")
        for d in odates:
            H.append(
                '<th colspan="2" width="100px" style="padding-left: 5px; padding-right: 5px;">'
                + d.strftime("%d/%m/%Y")
                + "</th>"
            )
        H.append("</tr><tr><td>&nbsp;</td>")
        H.append("<th>AM</th><th>PM</th>" * len(dates))
        H.append("</tr>")
        #
        if not etuds:
            H.append(
                '<tr><td><span class="redboldtext">Aucun étudiant inscrit !</span></td></tr>'
            )
        i = 1
        cnx = self.GetDBConnexion()
        cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
        for etud in etuds:
            i += 1
            etudid = etud["etudid"]
            # UE capitalisee dans semestre courant ?
            cap = []
            if etud["cursem"]:
                nt = self.Notes._getNotesCache().get_NotesTable(
                    self.Notes, etud["cursem"]["formsemestre_id"]
                )  # > get_ues, get_etud_ue_status
                for ue in nt.get_ues():
                    status = nt.get_etud_ue_status(etudid, ue["ue_id"])
                    if status["is_capitalized"]:
                        cap.append(ue["acronyme"])
            if cap:
                capstr = ' <span class="capstr">(%s cap.)</span>' % ", ".join(cap)
            else:
                capstr = ""

            tr_class = ("row_1", "row_2", "row_3")[i % 3]
            td_matin_class = ("matin_1", "matin_2", "matin_3")[i % 3]

            H.append(
                '<tr class="%s"><td><b class="etudinfo" id="%s"><a class="discretelink" href="ficheEtud?etudid=%s" target="new">%s</a></b>%s</td>'
                % (tr_class, etudid, etudid, etud["nomprenom"], capstr)
            )
            etud_abs = self.ListAbsInRange(
                etudid, begin, end, moduleimpl_id=moduleimpl_id, cursor=cursor
            )
            for d in odates:
                date = d.strftime("%Y-%m-%d")
                # matin
                is_abs = {"jour": d, "matin": True} in etud_abs
                if is_abs:
                    checked = "checked"
                else:
                    checked = ""
                #  bulle lors du passage souris
                coljour = sco_abs.DAYNAMES[(calendar.weekday(d.year, d.month, d.day))]
                datecol = coljour + " " + d.strftime("%d/%m/%Y")
                bulle_am = '"' + etud["nomprenom"] + " - " + datecol + ' (matin)"'
                bulle_pm = '"' + etud["nomprenom"] + " - " + datecol + ' (ap.midi)"'

                H.append(
                    '<td class="%s"><a title=%s><input type="checkbox" name="abslist:list" value="%s" %s onclick="on_toggled(this, \'%s\', \'%s\')"/></a></td>'
                    % (
                        td_matin_class,
                        bulle_am,
                        etudid + ":" + date + ":" + "am",
                        checked,
                        etudid,
                        date + ":am",
                    )
                )
                # après-midi
                is_abs = {"jour": d, "matin": False} in etud_abs
                if is_abs:
                    checked = "checked"
                else:
                    checked = ""
                H.append(
                    '<td><a title=%s><input type="checkbox" name="abslist:list" value="%s" %s onclick="on_toggled(this, \'%s\', \'%s\')"/></a></td>'
                    % (
                        bulle_pm,
                        etudid + ":" + date + ":" + "pm",
                        checked,
                        etudid,
                        date + ":pm",
                    )
                )
            H.append("</tr>")
        H.append("</table>")
        # place la liste des etudiants et les dates pour pouvoir effacer les absences
        H.append(
            '<input type="hidden" name="etudids" value="%s"/>'
            % ",".join([etud["etudid"] for etud in etuds])
        )
        H.append('<input type="hidden" name="datedebut" value="%s"/>' % dates[0])
        H.append('<input type="hidden" name="datefin" value="%s"/>' % dates[-1])
        H.append('<input type="hidden" name="dates" value="%s"/>' % ",".join(dates))
        H.append(
            '<input type="hidden" name="destination" value="%s"/>'
            % urllib.quote(destination)
        )
        #
        # version pour formulaire avec AJAX (Yann LB)
        H.append(
            """
            <p><input type="button" value="Retour" onClick="window.location='%s'"/>
            </p>
            </form>
            </p>
            </td></tr></table>
            <p class="help">Les cases cochées correspondent à des absences.
            Les absences saisies ne sont pas justifiées (sauf si un justificatif a été entré
            par ailleurs).
            </p><p class="help">Si vous "décochez" une case,  l'absence correspondante sera supprimée.
            Attention, les modifications sont automatiquement entregistrées au fur et à mesure.
            </p>
        """
            % destination
        )
        return H

    def _TablesAbsEtud(
        self,
        etudid,
        datedebut,
        with_evals=True,
        format="html",
        absjust_only=0,
        REQUEST=None,
    ):
        """Tables des absences justifiees et non justifiees d'un étudiant sur l'année en cours"""
        absjust = self.ListeAbsJust(etudid=etudid, datedebut=datedebut)
        absnonjust = self.ListeAbsNonJust(etudid=etudid, datedebut=datedebut)
        # examens ces jours là ?
        if with_evals:
            cnx = self.GetDBConnexion()
            cursor = cnx.cursor(cursor_factory=notesdb.ScoDocCursor)
            for a in absnonjust + absjust:
                cursor.execute(
                    """select eval.*
                from notes_evaluation eval, notes_moduleimpl_inscription mi, notes_moduleimpl m
                where eval.jour = %(jour)s and eval.moduleimpl_id = m.moduleimpl_id
                and mi.moduleimpl_id = m.moduleimpl_id and mi.etudid = %(etudid)s""",
                    {"jour": a["jour"].strftime("%Y-%m-%d"), "etudid": etudid},
                )
                a["evals"] = cursor.dictfetchall()
                cursor.execute(
                    """SELECT mi.moduleimpl_id
                from  absences abs, notes_moduleimpl_inscription mi, notes_moduleimpl m
                where abs.matin = %(matin)s and abs.jour = %(jour)s and abs.etudid=%(etudid)s and abs.moduleimpl_id=mi.moduleimpl_id and mi.moduleimpl_id=m.moduleimpl_id 
                and mi.etudid = %(etudid)s""",
                    {
                        "matin": bool(a["matin"]),
                        "jour": a["jour"].strftime("%Y-%m-%d"),
                        "etudid": etudid,
                    },
                )
                a["absent"] = cursor.dictfetchall()

        def matin(x):
            if x:
                return "matin"
            else:
                return "après-midi"

        def descr_exams(a):
            if not a.has_key("evals"):
                return ""
            ex = []
            for ev in a["evals"]:
                mod = sco_moduleimpl.do_moduleimpl_withmodule_list(
                    self.Notes, moduleimpl_id=ev["moduleimpl_id"]
                )[0]
                if format == "html":
                    ex.append(
                        '<a href="Notes/moduleimpl_status?moduleimpl_id=%s">%s</a>'
                        % (mod["moduleimpl_id"], mod["module"]["code"])
                    )
                else:
                    ex.append(mod["module"]["code"])
            if ex:
                return ", ".join(ex)
            return ""

        def descr_abs(a):
            ex = []
            for ev in a.get("absent", []):
                mod = sco_moduleimpl.do_moduleimpl_withmodule_list(
                    self.Notes, moduleimpl_id=ev["moduleimpl_id"]
                )[0]
                if format == "html":
                    ex.append(
                        '<a href="Notes/moduleimpl_status?moduleimpl_id=%s">%s</a>'
                        % (mod["moduleimpl_id"], mod["module"]["code"])
                    )
                else:
                    ex.append(mod["module"]["code"])
            if ex:
                return ", ".join(ex)
            return ""

        # ajoute date formatée et évaluations
        for L in (absnonjust, absjust):
            for a in L:
                if with_evals:
                    a["exams"] = descr_exams(a)
                a["datedmy"] = a["jour"].strftime("%d/%m/%Y")
                a["ampm"] = int(a["matin"])
                a["matin"] = matin(a["matin"])
                index = a["description"].find(")")
                if index != -1:
                    a["motif"] = a["description"][1:index]
                else:
                    a["motif"] = ""
                a["description"] = descr_abs(a) or ""

        # ajoute lien pour justifier
        if format == "html":
            for a in absnonjust:
                a["justlink"] = "<em>justifier</em>"
                a["_justlink_target"] = (
                    "doJustifAbsence?etudid=%s&amp;datedebut=%s&amp;datefin=%s&amp;demijournee=%s"
                    % (etudid, a["datedmy"], a["datedmy"], a["ampm"])
                )
        #
        titles = {
            "datedmy": "Date",
            "matin": "",
            "exams": "Examens ce jour",
            "justlink": "",
            "description": "Modules",
            "motif": "Motif",
        }
        columns_ids = ["datedmy", "matin"]
        if format in ("json", "xml"):
            columns_ids += ["jour", "ampm"]
        if with_evals:
            columns_ids.append("exams")

        columns_ids.append("description")
        columns_ids.append("motif")
        if format == "html":
            columns_ids.append("justlink")

        return titles, columns_ids, absnonjust, absjust

    security.declareProtected(ScoView, "EtatAbsencesGr")  # ported from dtml

    def EtatAbsencesGr(
        self,
        group_ids=[],  # list of groups to display
        debut="",
        fin="",
        with_boursier=True,  # colonne boursier
        format="html",
        REQUEST=None,
    ):
        """Liste les absences de groupes"""
        datedebut = notesdb.DateDMYtoISO(debut)
        datefin = notesdb.DateDMYtoISO(fin)
        # Informations sur les groupes à afficher:
        groups_infos = sco_groups_view.DisplayedGroupsInfos(
            self, group_ids, REQUEST=REQUEST
        )
        formsemestre_id = groups_infos.formsemestre_id
        sem = groups_infos.formsemestre

        # Construit tableau (etudid, statut, nomprenom, nbJust, nbNonJust, NbTotal)
        T = []
        for m in groups_infos.members:
            etud = self.getEtudInfo(etudid=m["etudid"], filled=True)[0]
            nbabs = self.CountAbs(etudid=etud["etudid"], debut=datedebut, fin=datefin)
            nbabsjust = self.CountAbsJust(
                etudid=etud["etudid"], debut=datedebut, fin=datefin
            )
            nbjustifs_noabs = len(
                self.ListeJustifs(
                    etudid=etud["etudid"], datedebut=datedebut, only_no_abs=True
                )
            )
            # retrouve sem dans etud['sems']
            s = None
            for s in etud["sems"]:
                if s["formsemestre_id"] == formsemestre_id:
                    break
            if not s or s["formsemestre_id"] != formsemestre_id:
                raise ValueError(
                    "EtatAbsencesGr: can't retreive sem"
                )  # bug or malicious arg
            T.append(
                {
                    "etudid": etud["etudid"],
                    "etatincursem": s["ins"]["etat"],
                    "nomprenom": etud["nomprenom"],
                    "nbabsjust": nbabsjust,
                    "nbabsnonjust": nbabs - nbabsjust,
                    "nbabs": nbabs,
                    "nbjustifs_noabs": nbjustifs_noabs,
                    "_nomprenom_target": "CalAbs?etudid=%s" % etud["etudid"],
                    "_nomprenom_td_attrs": 'id="%s" class="etudinfo"' % etud["etudid"],
                    "boursier": etud["boursier"],
                }
            )
            if s["ins"]["etat"] == "D":
                T[-1]["_css_row_class"] = "etuddem"
                T[-1]["nomprenom"] += " (dem)"
        columns_ids = [
            "nomprenom",
            "nbjustifs_noabs",
            "nbabsjust",
            "nbabsnonjust",
            "nbabs",
        ]
        if with_boursier:
            columns_ids[1:1] = ["boursier"]
        if groups_infos.tous_les_etuds_du_sem:
            gr_tit = ""
        else:
            if len(groups_infos.group_ids) > 1:
                p = "des groupes"
            else:
                p = "du groupe"
            if format == "html":
                h = ' <span class="fontred">' + groups_infos.groups_titles + "</span>"
            else:
                h = groups_infos.groups_titles
            gr_tit = p + h

        title = "Etat des absences %s" % gr_tit
        if format == "xls" or format == "xml":
            columns_ids = ["etudid"] + columns_ids
        tab = GenTable(
            columns_ids=columns_ids,
            rows=T,
            preferences=self.get_preferences(formsemestre_id),
            titles={
                "etatincursem": "Etat",
                "nomprenom": "Nom",
                "nbabsjust": "Justifiées",
                "nbabsnonjust": "Non justifiées",
                "nbabs": "Total",
                "nbjustifs_noabs": "Justifs non utilisés",
                "boursier": "Bourse",
            },
            html_sortable=True,
            html_class="table_leftalign",
            html_header=self.sco_header(
                REQUEST,
                page_title=title,
                init_qtip=True,
                javascripts=["js/etud_info.js"],
            ),
            html_title=self.Notes.html_sem_header(
                REQUEST, "%s" % title, sem, with_page_header=False
            )
            + "<p>Période du %s au %s (nombre de <b>demi-journées</b>)<br/>"
            % (debut, fin),
            base_url="%s&amp;formsemestre_id=%s&amp;debut=%s&amp;fin=%s"
            % (groups_infos.base_url, formsemestre_id, debut, fin),
            filename="etat_abs_"
            + scu.make_filename(
                "%s de %s" % (groups_infos.groups_filename, sem["titreannee"])
            ),
            caption=title,
            html_next_section="""</table>
<p class="help">
Justifs non utilisés: nombre de demi-journées avec justificatif mais sans absences relevées.
</p>
<p class="help">
Cliquez sur un nom pour afficher le calendrier des absences<br/>
ou entrez une date pour visualiser les absents un jour donné&nbsp;:
</p>
<div style="margin-bottom: 10px;">
<form action="EtatAbsencesDate" method="get" action="%s">
<input type="hidden" name="formsemestre_id" value="%s">
%s
<input type="text" name="date" size="10" class="datepicker"/>
<input type="submit" name="" value="visualiser les absences">
</form></div>
                        """
            % (REQUEST.URL0, formsemestre_id, groups_infos.get_form_elem()),
        )
        return tab.make_page(self, format=format, REQUEST=REQUEST)

    security.declareProtected(ScoView, "EtatAbsencesDate")  # ported from dtml

    def EtatAbsencesDate(
        self, group_ids=[], date=None, REQUEST=None  # list of groups to display
    ):
        """Etat des absences pour un groupe à une date donnée"""
        # Informations sur les groupes à afficher:
        groups_infos = sco_groups_view.DisplayedGroupsInfos(
            self, group_ids, REQUEST=REQUEST
        )
        H = [self.sco_header(page_title="Etat des absences", REQUEST=REQUEST)]
        if date:
            dateiso = notesdb.DateDMYtoISO(date)
            nbetud = 0
            t_nbabsjustam = 0
            t_nbabsam = 0
            t_nbabsjustpm = 0
            t_nbabspm = 0
            H.append("<h2>État des absences le %s</h2>" % date)
            H.append(
                """<table border="0" cellspacing="4" cellpadding="0">
             <tr><th>&nbsp;</th>
            <th style="width: 10em;">Matin</th><th style="width: 10em;">Après-midi</th></tr>
            """
            )
            for etud in groups_infos.members:
                nbabsam = self.CountAbs(
                    etudid=etud["etudid"], debut=dateiso, fin=dateiso, matin=1
                )
                nbabspm = self.CountAbs(
                    etudid=etud["etudid"], debut=dateiso, fin=dateiso, matin=0
                )
                if (nbabsam != 0) or (nbabspm != 0):
                    nbetud += 1
                    nbabsjustam = self.CountAbsJust(
                        etudid=etud["etudid"], debut=dateiso, fin=dateiso, matin=1
                    )
                    nbabsjustpm = self.CountAbsJust(
                        etudid=etud["etudid"], debut=dateiso, fin=dateiso, matin=0
                    )
                    H.append(
                        """<tr bgcolor="#FFFFFF"><td>
                     <a href="CalAbs?etudid=%(etudid)s"><font color="#A00000">%(nomprenom)s</font></a></td><td align="center">"""
                        % etud
                    )  # """
                    if nbabsam != 0:
                        if nbabsjustam:
                            H.append("Just.")
                            t_nbabsjustam += 1
                        else:
                            H.append("Abs.")
                            t_nbabsam += 1
                    else:
                        H.append("")
                    H.append('</td><td align="center">')
                    if nbabspm != 0:
                        if nbabsjustpm:
                            H.append("Just.")
                            t_nbabsjustam += 1
                        else:
                            H.append("Abs.")
                            t_nbabspm += 1
                    else:
                        H.append("")
                    H.append("</td></tr>")
            H.append(
                """<tr bgcolor="#FFFFFF"><td></td><td>%d abs, %d just.</td><td>%d abs, %d just.</td></tr>"""
                % (t_nbabsam, t_nbabsjustam, t_nbabspm, t_nbabsjustpm)
            )
            H.append("</table>")
            if nbetud == 0:
                H.append("<p>Aucune absence !</p>")
        else:
            H.append(
                """<h2>Erreur: vous n'avez pas choisi de date !</h2>
              <a class="stdlink" href="%s">Continuer</a>"""
                % REQUEST.HTTP_REFERER
            )

        return "\n".join(H) + self.sco_footer(REQUEST)

    # ----- Gestion des "billets d'absence": signalement par les etudiants eux mêmes (à travers le portail)
    security.declareProtected(ScoAbsAddBillet, "AddBilletAbsence")

    def AddBilletAbsence(
        self,
        begin,
        end,
        description,
        etudid=False,
        code_nip=None,
        code_ine=None,
        justified=True,
        REQUEST=None,
        xml_reply=True,
    ):
        """Memorise un "billet"
        begin et end sont au format ISO (eg "1999-01-08 04:05:06")
        """
        t0 = time.time()
        # check etudid
        etuds = self.getEtudInfo(
            etudid=etudid, code_nip=code_nip, REQUEST=REQUEST, filled=True
        )
        if not etuds:
            return scu.log_unknown_etud(self, REQUEST=REQUEST)
        etud = etuds[0]
        # check dates
        begin_date = dateutil.parser.isoparse(begin)  # may raises ValueError
        end_date = dateutil.parser.isoparse(end)
        if begin_date > end_date:
            raise ValueError("invalid dates")
        #
        justified = int(justified)
        #
        cnx = self.GetDBConnexion()
        billet_id = sco_abs.billet_absence_create(
            cnx,
            {
                "etudid": etud["etudid"],
                "abs_begin": begin,
                "abs_end": end,
                "description": description,
                "etat": 0,
                "justified": justified,
            },
        )
        if xml_reply:
            # Renvoie le nouveau billet en XML
            if REQUEST:
                REQUEST.RESPONSE.setHeader("content-type", scu.XML_MIMETYPE)

            billets = sco_abs.billet_absence_list(cnx, {"billet_id": billet_id})
            tab = self._tableBillets(billets, etud=etud)
            log(
                "AddBilletAbsence: new billet_id=%s (%gs)"
                % (billet_id, time.time() - t0)
            )
            return tab.make_page(self, REQUEST=REQUEST, format="xml")
        else:
            return billet_id

    security.declareProtected(ScoAbsAddBillet, "AddBilletAbsenceForm")

    def AddBilletAbsenceForm(self, etudid, REQUEST=None):
        """Formulaire ajout billet (pour tests seulement, le vrai formulaire accessible aux etudiants
        étant sur le portail étudiant).
        """
        etud = self.getEtudInfo(etudid=etudid, filled=1, REQUEST=REQUEST)[0]
        H = [
            self.sco_header(
                REQUEST, page_title="Billet d'absence de %s" % etud["nomprenom"]
            )
        ]
        tf = TrivialFormulator(
            REQUEST.URL0,
            REQUEST.form,
            (
                ("etudid", {"input_type": "hidden"}),
                ("begin", {"input_type": "date"}),
                ("end", {"input_type": "date"}),
                (
                    "justified",
                    {"input_type": "boolcheckbox", "default": 0, "title": "Justifiée"},
                ),
                ("description", {"input_type": "textarea"}),
            ),
        )
        if tf[0] == 0:
            return "\n".join(H) + tf[1] + self.sco_footer(REQUEST)
        elif tf[0] == -1:
            return REQUEST.RESPONSE.redirect(self.ScoURL())
        else:
            e = tf[2]["begin"].split("/")
            begin = e[2] + "-" + e[1] + "-" + e[0] + " 00:00:00"
            e = tf[2]["end"].split("/")
            end = e[2] + "-" + e[1] + "-" + e[0] + " 00:00:00"
            log(
                self.AddBilletAbsence(
                    begin,
                    end,
                    tf[2]["description"],
                    etudid=etudid,
                    xml_reply=True,
                    justified=tf[2]["justified"],
                )
            )
            return REQUEST.RESPONSE.redirect("listeBilletsEtud?etudid=" + etudid)

    def _tableBillets(self, billets, etud=None, title=""):
        for b in billets:
            if b["abs_begin"].hour < 12:
                m = " matin"
            else:
                m = " après-midi"
            b["abs_begin_str"] = b["abs_begin"].strftime("%d/%m/%Y") + m
            if b["abs_end"].hour < 12:
                m = " matin"
            else:
                m = " après-midi"
            b["abs_end_str"] = b["abs_end"].strftime("%d/%m/%Y") + m
            if b["etat"] == 0:
                if b["justified"] == 0:
                    b["etat_str"] = "à traiter"
                else:
                    b["etat_str"] = "à justifier"
                b["_etat_str_target"] = (
                    "ProcessBilletAbsenceForm?billet_id=%s" % b["billet_id"]
                )
                if etud:
                    b["_etat_str_target"] += "&amp;etudid=%s" % etud["etudid"]
                b["_billet_id_target"] = b["_etat_str_target"]
            else:
                b["etat_str"] = "ok"
            if not etud:
                # ajoute info etudiant
                e = self.getEtudInfo(etudid=b["etudid"], filled=1)
                if not e:
                    b["nomprenom"] = "???"  # should not occur
                else:
                    b["nomprenom"] = e[0]["nomprenom"]
                b["_nomprenom_target"] = "ficheEtud?etudid=%s" % b["etudid"]
        if etud and not title:
            title = "Billets d'absence déclarés par %(nomprenom)s" % etud
        else:
            title = title
        columns_ids = ["billet_id"]
        if not etud:
            columns_ids += ["nomprenom"]
        columns_ids += ["abs_begin_str", "abs_end_str", "description", "etat_str"]

        tab = GenTable(
            titles={
                "billet_id": "Numéro",
                "abs_begin_str": "Début",
                "abs_end_str": "Fin",
                "description": "Raison de l'absence",
                "etat_str": "Etat",
            },
            columns_ids=columns_ids,
            page_title=title,
            html_title="<h2>%s</h2>" % title,
            preferences=self.get_preferences(),
            rows=billets,
            html_sortable=True,
        )
        return tab

    security.declareProtected(ScoView, "listeBilletsEtud")

    def listeBilletsEtud(self, etudid=False, REQUEST=None, format="html"):
        """Liste billets pour un etudiant"""
        etuds = self.getEtudInfo(etudid=etudid, filled=1, REQUEST=REQUEST)
        if not etuds:
            return scu.log_unknown_etud(self, format=format, REQUEST=REQUEST)

        etud = etuds[0]
        cnx = self.GetDBConnexion()
        billets = sco_abs.billet_absence_list(cnx, {"etudid": etud["etudid"]})
        tab = self._tableBillets(billets, etud=etud)
        return tab.make_page(self, REQUEST=REQUEST, format=format)

    security.declareProtected(ScoView, "XMLgetBilletsEtud")

    def XMLgetBilletsEtud(self, etudid=False, REQUEST=None):
        """Liste billets pour un etudiant"""
        if not self.get_preference("handle_billets_abs"):
            return ""
        t0 = time.time()
        r = self.listeBilletsEtud(etudid, REQUEST=REQUEST, format="xml")
        log("XMLgetBilletsEtud (%gs)" % (time.time() - t0))
        return r

    security.declareProtected(ScoView, "listeBillets")

    def listeBillets(self, REQUEST=None):
        """Page liste des billets non traités et formulaire recherche d'un billet"""
        cnx = self.GetDBConnexion()
        billets = sco_abs.billet_absence_list(cnx, {"etat": 0})
        tab = self._tableBillets(billets)
        T = tab.html()
        H = [
            self.sco_header(REQUEST, page_title="Billet d'absence non traités"),
            "<h2>Billets d'absence en attente de traitement (%d)</h2>" % len(billets),
        ]

        tf = TrivialFormulator(
            REQUEST.URL0,
            REQUEST.form,
            (("billet_id", {"input_type": "text", "title": "Numéro du billet"}),),
            submitbutton=False,
        )
        if tf[0] == 0:
            return "\n".join(H) + tf[1] + T + self.sco_footer(REQUEST)
        else:
            return REQUEST.RESPONSE.redirect(
                "ProcessBilletAbsenceForm?billet_id=" + tf[2]["billet_id"]
            )

    security.declareProtected(ScoAbsChange, "deleteBilletAbsence")

    def deleteBilletAbsence(self, billet_id, REQUEST=None, dialog_confirmed=False):
        """Supprime un billet."""
        cnx = self.GetDBConnexion()
        billets = sco_abs.billet_absence_list(cnx, {"billet_id": billet_id})
        if not billets:
            return REQUEST.RESPONSE.redirect(
                "listeBillets?head_message=Billet%%20%s%%20inexistant !" % billet_id
            )
        if not dialog_confirmed:
            tab = self._tableBillets(billets)
            return self.confirmDialog(
                """<h2>Supprimer ce billet ?</h2>""" + tab.html(),
                dest_url="",
                REQUEST=REQUEST,
                cancel_url="listeBillets",
                parameters={"billet_id": billet_id},
            )

        sco_abs.billet_absence_delete(cnx, billet_id)

        return REQUEST.RESPONSE.redirect("listeBillets?head_message=Billet%20supprimé")

    def _ProcessBilletAbsence(self, billet, estjust, description, REQUEST):
        """Traite un billet: ajoute absence(s) et éventuellement justificatifs,
        et change l'état du billet à 1.
        NB: actuellement, les heures ne sont utilisées que pour déterminer si matin et/ou après-midi.
        """
        cnx = self.GetDBConnexion()
        if billet["etat"] != 0:
            log("billet=%s" % billet)
            log("billet deja traité !")
            return -1
        n = 0  # nombre de demi-journées d'absence ajoutées
        # 1-- ajout des absences (et justifs)
        datedebut = billet["abs_begin"].strftime("%d/%m/%Y")
        datefin = billet["abs_end"].strftime("%d/%m/%Y")
        dates = sco_abs.DateRangeISO(self, datedebut, datefin)
        # commence après-midi ?
        if dates and billet["abs_begin"].hour > 11:
            self._AddAbsence(
                billet["etudid"], dates[0], 0, estjust, REQUEST, description=description
            )
            n += 1
            dates = dates[1:]
        # termine matin ?
        if dates and billet["abs_end"].hour < 12:
            self._AddAbsence(
                billet["etudid"],
                dates[-1],
                1,
                estjust,
                REQUEST,
                description=description,
            )
            n += 1
            dates = dates[:-1]

        for jour in dates:
            self._AddAbsence(
                billet["etudid"], jour, 0, estjust, REQUEST, description=description
            )
            self._AddAbsence(
                billet["etudid"], jour, 1, estjust, REQUEST, description=description
            )
            n += 2

        # 2- change etat du billet
        sco_abs.billet_absence_edit(cnx, {"billet_id": billet["billet_id"], "etat": 1})

        return n

    security.declareProtected(ScoAbsChange, "ProcessBilletAbsenceForm")

    def ProcessBilletAbsenceForm(self, billet_id, REQUEST=None):
        """Formulaire traitement d'un billet"""
        cnx = self.GetDBConnexion()
        billets = sco_abs.billet_absence_list(cnx, {"billet_id": billet_id})
        if not billets:
            return REQUEST.RESPONSE.redirect(
                "listeBillets?head_message=Billet%%20%s%%20inexistant !" % billet_id
            )
        billet = billets[0]
        etudid = billet["etudid"]
        etud = self.getEtudInfo(etudid=etudid, filled=1, REQUEST=REQUEST)[0]

        H = [
            self.sco_header(
                REQUEST,
                page_title="Traitement billet d'absence de %s" % etud["nomprenom"],
            ),
            '<h2>Traitement du billet %s : <a class="discretelink" href="ficheEtud?etudid=%s">%s</a></h2>'
            % (billet_id, etudid, etud["nomprenom"]),
        ]

        tf = TrivialFormulator(
            REQUEST.URL0,
            REQUEST.form,
            (
                ("billet_id", {"input_type": "hidden"}),
                (
                    "etudid",
                    {"input_type": "hidden"},
                ),  # pour centrer l'UI sur l'étudiant
                (
                    "estjust",
                    {"input_type": "boolcheckbox", "title": "Absences justifiées"},
                ),
                ("description", {"input_type": "text", "size": 42, "title": "Raison"}),
            ),
            initvalues={
                "description": billet["description"],
                "estjust": billet["justified"],
                "etudid": etudid,
            },
            submitlabel="Enregistrer ces absences",
        )
        if tf[0] == 0:
            tab = self._tableBillets([billet], etud=etud)
            H.append(tab.html())
            if billet["justified"] == 1:
                H.append(
                    """<p>L'étudiant pense pouvoir justifier cette absence.<br/><em>Vérifiez le justificatif avant d'enregistrer.</em></p>"""
                )
            F = (
                """<p><a class="stdlink" href="deleteBilletAbsence?billet_id=%s">Supprimer ce billet</a> (utiliser en cas d'erreur, par ex. billet en double)</p>"""
                % billet_id
            )
            F += '<p><a class="stdlink" href="listeBillets">Liste de tous les billets en attente</a></p>'

            return "\n".join(H) + "<br/>" + tf[1] + F + self.sco_footer(REQUEST)
        elif tf[0] == -1:
            return REQUEST.RESPONSE.redirect(self.ScoURL())
        else:
            n = self._ProcessBilletAbsence(
                billet, tf[2]["estjust"], tf[2]["description"], REQUEST
            )
            if tf[2]["estjust"]:
                j = "justifiées"
            else:
                j = "non justifiées"
            H.append('<div class="head_message">')
            if n > 0:
                H.append("%d absences (1/2 journées) %s ajoutées" % (n, j))
            elif n == 0:
                H.append("Aucun jour d'absence dans les dates indiquées !")
            elif n < 0:
                H.append("Ce billet avait déjà été traité !")
            H.append(
                '</div><p><a class="stdlink" href="listeBillets">Autre billets en attente</a></p><h4>Billets déclarés par %s</h4>'
                % (etud["nomprenom"])
            )
            billets = sco_abs.billet_absence_list(cnx, {"etudid": etud["etudid"]})
            tab = self._tableBillets(billets, etud=etud)
            H.append(tab.html())
            return "\n".join(H) + self.sco_footer(REQUEST)

    security.declareProtected(ScoView, "XMLgetAbsEtud")

    def XMLgetAbsEtud(self, beg_date="", end_date="", REQUEST=None):
        """returns list of absences in date interval"""
        t0 = time.time()
        etud = self.getEtudInfo(REQUEST=REQUEST)[0]
        exp = re.compile(r"^(\d{4})\D?(0[1-9]|1[0-2])\D?([12]\d|0[1-9]|3[01])$")
        if not exp.match(beg_date):
            raise ScoValueError("invalid date: %s" % beg_date)
        if not exp.match(end_date):
            raise ScoValueError("invalid date: %s" % end_date)

        Abs = self._ListeAbsDate(etud["etudid"], beg_date, end_date)

        REQUEST.RESPONSE.setHeader("content-type", scu.XML_MIMETYPE)
        doc = jaxml.XML_document(encoding=scu.SCO_ENCODING)
        doc.absences(etudid=etud["etudid"], beg_date=beg_date, end_date=end_date)
        doc._push()
        for a in Abs:
            if a["estabs"]:  # ne donne pas les justifications si pas d'absence
                doc._push()
                doc.abs(
                    begin=a["begin"],
                    end=a["end"],
                    description=a["description"],
                    justified=a["estjust"],
                )
                doc._pop()
        doc._pop()
        log("XMLgetAbsEtud (%gs)" % (time.time() - t0))
        return repr(doc)