MonScoDocEssai/app/scodoc/sco_bulletins.py

# -*- mode: python -*-
# -*- coding: utf-8 -*-

##############################################################################
#
# Gestion scolarite IUT
#
# Copyright (c) 1999 - 2023 Emmanuel Viennet.  All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#   Emmanuel Viennet      emmanuel.viennet@viennet.net
#
##############################################################################

"""Génération des bulletins de notes

"""
import email
import time
import numpy as np

from flask import g, request
from flask import flash, jsonify, render_template, url_for
from flask_login import current_user

from app import email
from app import log
from app.scodoc.sco_utils import json_error
from app.but import bulletin_but
from app.comp import res_sem
from app.comp.res_compat import NotesTableCompat
from app.models import FormSemestre, Identite, ModuleImplInscription
from app.scodoc.sco_permissions import Permission
from app.scodoc.sco_exceptions import AccessDenied, ScoValueError
from app.scodoc import html_sco_header
from app.scodoc import htmlutils
from app.scodoc import sco_abs
from app.scodoc import sco_abs_views
from app.scodoc import sco_bulletins_generator
from app.scodoc import sco_bulletins_json
from app.scodoc import sco_bulletins_pdf
from app.scodoc import sco_bulletins_xml
from app.scodoc import codes_cursus
from app.scodoc import sco_etud
from app.scodoc import sco_evaluation_db
from app.scodoc import sco_formations
from app.scodoc import sco_formsemestre
from app.scodoc import sco_groups
from app.scodoc import sco_permissions_check
from app.scodoc import sco_preferences
from app.scodoc import sco_pvjury
from app.scodoc import sco_users
import app.scodoc.sco_utils as scu
from app.scodoc.sco_utils import ModuleType, fmt_note
import app.scodoc.notesdb as ndb


def get_formsemestre_bulletin_etud_json(
    formsemestre: FormSemestre,
    etud: Identite,
    force_publishing=False,
    version="long",
) -> str:
    """Le JSON du bulletin d'un étudiant, quel que soit le type de formation."""
    if formsemestre.formation.is_apc():
        bul = bulletin_but.BulletinBUT(formsemestre)
        if not etud.id in bul.res.identdict:
            return json_error(404, "get_formsemestre_bulletin_etud_json: invalid etud")
        return jsonify(
            bul.bulletin_etud(
                etud,
                formsemestre,
                force_publishing=force_publishing,
                version=version,
            )
        )
    return formsemestre_bulletinetud(
        etud,
        formsemestre_id=formsemestre.id,
        format="json",
        version=version,
        xml_with_decisions=True,
        force_publishing=force_publishing,
    )


# -------------
def make_context_dict(formsemestre: FormSemestre, etud: dict) -> dict:
    """Construit dictionnaire avec valeurs pour substitution des textes
    (preferences bul_pdf_*)
    """
    C = formsemestre.get_infos_dict()
    C["responsable"] = formsemestre.responsables_str()
    C["anneesem"] = C["annee"]  # backward compat
    C.update(etud)
    # copie preferences
    for name in sco_preferences.get_base_preferences().prefs_name:
        C[name] = sco_preferences.get_preference(name, formsemestre.id)

    # ajoute groupes et group_0, group_1, ...
    sco_groups.etud_add_group_infos(etud, formsemestre.id)
    C["groupes"] = etud["groupes"]
    n = 0
    for partition_id in etud["partitions"]:
        C["group_%d" % n] = etud["partitions"][partition_id]["group_name"]
        n += 1

    # ajoute date courante
    t = time.localtime()
    C["date_dmy"] = time.strftime("%d/%m/%Y", t)
    C["date_iso"] = time.strftime("%Y-%m-%d", t)

    return C


def formsemestre_bulletinetud_dict(formsemestre_id, etudid, version="long"):
    """Collecte informations pour bulletin de notes
    Retourne un dictionnaire (avec valeur par défaut chaine vide).
    Le contenu du dictionnaire dépend des options (rangs, ...)
    et de la version choisie (short, long, selectedevals).

    Cette fonction est utilisée pour les bulletins CLASSIQUES (DUT, ...)
    en HTML et PDF, mais pas ceux en XML.
    """
    from app.scodoc import sco_abs

    if not version in scu.BULLETINS_VERSIONS:
        raise ValueError("invalid version code !")

    prefs = sco_preferences.SemPreferences(formsemestre_id)
    formsemestre = FormSemestre.query.get_or_404(formsemestre_id)
    nt: NotesTableCompat = res_sem.load_formsemestre_results(formsemestre)
    if not nt.get_etud_etat(etudid):
        raise ScoValueError("Étudiant non inscrit à ce semestre")
    I = scu.DictDefault(defaultvalue="")
    I["etudid"] = etudid
    I["formsemestre_id"] = formsemestre_id
    I["sem"] = formsemestre.get_infos_dict()
    I["server_name"] = request.url_root

    # Formation et parcours
    formation_dict = None
    if I["sem"]["formation_id"]:
        formation_dicts = sco_formations.formation_list(
            args={"formation_id": I["sem"]["formation_id"]}
        )
        if formation_dicts:
            formation_dict = formation_dicts[0]
    if formation_dict is None:  # what's the fuck ?
        formation_dict = {
            "acronyme": "?",
            "code_specialite": "",
            "dept_id": 1,
            "formation_code": "?",
            "formation_id": -1,
            "id": -1,
            "referentiel_competence_id": None,
            "titre": "?",
            "titre_officiel": "?",
            "type_parcours": 0,
            "version": 0,
        }
    I["formation"] = formation_dict
    I["parcours"] = codes_cursus.get_cursus_from_code(I["formation"]["type_parcours"])
    # Infos sur l'etudiant
    I["etud"] = sco_etud.get_etud_info(etudid=etudid, filled=True)[0]
    I["descr_situation"] = I["etud"]["inscriptionstr"]
    if I["etud"]["inscription_formsemestre_id"]:
        I[
            "descr_situation_html"
        ] = f"""<a href="{url_for(
                "notes.formsemestre_status",
                scodoc_dept=g.scodoc_dept,
                formsemestre_id=I["etud"]["inscription_formsemestre_id"],
            )}">{I["descr_situation"]}</a>"""
    else:
        I["descr_situation_html"] = I["descr_situation"]
    # Groupes:
    partitions = sco_groups.get_partitions_list(formsemestre_id, with_default=False)
    partitions_etud_groups = {}  # { partition_id : { etudid : group } }
    for partition in partitions:
        pid = partition["partition_id"]
        partitions_etud_groups[pid] = sco_groups.get_etud_groups_in_partition(pid)
    # --- Absences
    I["nbabs"], I["nbabsjust"] = sco_abs.get_abs_count(etudid, nt.sem)

    # --- Decision Jury
    infos, dpv = etud_descr_situation_semestre(
        etudid,
        formsemestre_id,
        format="html",
        show_date_inscr=prefs["bul_show_date_inscr"],
        show_decisions=prefs["bul_show_decision"],
        show_uevalid=prefs["bul_show_uevalid"],
        show_mention=prefs["bul_show_mention"],
    )

    I.update(infos)

    I["etud_etat_html"] = _get_etud_etat_html(
        formsemestre.etuds_inscriptions[etudid].etat
    )
    I["etud_etat"] = nt.get_etud_etat(etudid)
    I["filigranne"] = sco_bulletins_pdf.get_filigranne(
        I["etud_etat"], prefs, decision_sem=I["decision_sem"]
    )
    I["demission"] = ""
    if I["etud_etat"] == scu.DEMISSION:
        I["demission"] = "(Démission)"
    elif I["etud_etat"] == codes_cursus.DEF:
        I["demission"] = "(Défaillant)"

    # --- Appreciations
    I.update(get_appreciations_list(formsemestre_id, etudid))

    # --- Notes
    ues = nt.get_ues_stat_dict()
    modimpls = nt.get_modimpls_dict()
    moy_gen = nt.get_etud_moy_gen(etudid)
    I["nb_inscrits"] = len(nt.etud_moy_gen_ranks)
    I["moy_gen"] = scu.fmt_note(moy_gen)
    I["moy_min"] = scu.fmt_note(nt.moy_min)
    I["moy_max"] = scu.fmt_note(nt.moy_max)
    I["mention"] = ""
    if dpv:
        decision_sem = dpv["decisions"][0]["decision_sem"]
        if decision_sem and codes_cursus.code_semestre_validant(decision_sem["code"]):
            I["mention"] = scu.get_mention(moy_gen)

    if dpv and dpv["decisions"][0]:
        I["sum_ects"] = dpv["decisions"][0]["sum_ects"]
        I["sum_ects_capitalises"] = dpv["decisions"][0]["sum_ects_capitalises"]
    else:
        I["sum_ects"] = 0
        I["sum_ects_capitalises"] = 0
    I["moy_moy"] = scu.fmt_note(nt.moy_moy)  # moyenne des moyennes generales
    if (not isinstance(moy_gen, str)) and (not isinstance(nt.moy_moy, str)):
        I["moy_gen_bargraph_html"] = "&nbsp;" + htmlutils.horizontal_bargraph(
            moy_gen * 5, nt.moy_moy * 5
        )
    else:
        I["moy_gen_bargraph_html"] = ""

    if prefs["bul_show_rangs"]:
        rang = str(nt.get_etud_rang(etudid))
    else:
        rang = ""

    rang_gr, ninscrits_gr, gr_name = get_etud_rangs_groups(
        etudid, partitions, partitions_etud_groups, nt
    )

    if nt.get_moduleimpls_attente():
        # n'affiche pas le rang sur le bulletin s'il y a des
        # notes en attente dans ce semestre
        rang = scu.RANG_ATTENTE_STR
        rang_gr = scu.DictDefault(defaultvalue=scu.RANG_ATTENTE_STR)
    inscriptions_counts = nt.get_inscriptions_counts()
    I["rang"] = rang
    I["rang_gr"] = rang_gr
    I["gr_name"] = gr_name
    I["ninscrits_gr"] = ninscrits_gr
    I["nbetuds"] = len(nt.etud_moy_gen_ranks)
    I["nb_demissions"] = inscriptions_counts[scu.DEMISSION]
    I["nb_defaillants"] = inscriptions_counts[scu.DEF]
    if prefs["bul_show_rangs"]:
        I["rang_nt"] = "%s / %d" % (
            rang,
            inscriptions_counts[scu.INSCRIT],
        )
        I["rang_txt"] = "Rang " + I["rang_nt"]
    else:
        I["rang_nt"], I["rang_txt"] = "", ""
    I["note_max"] = 20.0  # notes toujours sur 20
    I["bonus_sport_culture"] = nt.bonus[etudid] if nt.bonus is not None else 0.0
    # Liste les UE / modules /evals
    I["ues"] = []
    I["matieres_modules"] = {}
    I["matieres_modules_capitalized"] = {}
    for ue in ues:
        u = ue.copy()
        ue_status = nt.get_etud_ue_status(etudid, ue["ue_id"])
        if (
            ModuleImplInscription.nb_inscriptions_dans_ue(
                formsemestre_id, etudid, ue["ue_id"]
            )
            == 0
        ) and not ue_status["is_capitalized"]:
            # saute les UE où l'on est pas inscrit et n'avons pas de capitalisation
            continue

        u["ue_status"] = ue_status  # { 'moy', 'coef_ue', ...}
        if ue["type"] != codes_cursus.UE_SPORT:
            u["cur_moy_ue_txt"] = scu.fmt_note(ue_status["cur_moy_ue"])
        else:
            if nt.bonus is not None:
                x = scu.fmt_note(nt.bonus[etudid], keep_numeric=True)
            else:
                x = ""
            if isinstance(x, str):
                if nt.bonus_ues is None:
                    u["cur_moy_ue_txt"] = "pas de bonus"
                else:
                    u["cur_moy_ue_txt"] = "bonus appliqué sur les UEs"
            else:
                u["cur_moy_ue_txt"] = f"bonus de {fmt_note(x)} points"
            if nt.bonus_ues is not None:
                u["cur_moy_ue_txt"] += " (+ues)"
        u["moy_ue_txt"] = scu.fmt_note(ue_status["moy"])
        if ue_status["coef_ue"] != None:
            u["coef_ue_txt"] = scu.fmt_coef(ue_status["coef_ue"])
        else:
            u["coef_ue_txt"] = "-"

        if (
            dpv
            and dpv["decisions"][0]["decisions_ue"]
            and ue["ue_id"] in dpv["decisions"][0]["decisions_ue"]
        ):
            u["ects"] = dpv["decisions"][0]["decisions_ue"][ue["ue_id"]]["ects"]
            if ue["type"] == codes_cursus.UE_ELECTIVE:
                u["ects"] = (
                    "%g+" % u["ects"]
                )  # ajoute un "+" pour indiquer ECTS d'une UE élective
        else:
            if ue_status["is_capitalized"]:
                u["ects"] = ue_status["ue"].get("ects", "-")
            else:
                u["ects"] = "-"
        modules, ue_attente = _ue_mod_bulletin(
            etudid, formsemestre_id, ue["ue_id"], modimpls, nt, version
        )
        #
        u["modules"] = modules  # detail des modules de l'UE (dans le semestre courant)
        # auparavant on filtrait les modules sans notes
        #   si ue_status['cur_moy_ue'] != 'NA' alors u['modules'] = [] (pas de moyenne => pas de modules)

        u[
            "modules_capitalized"
        ] = []  # modules de l'UE capitalisée (liste vide si pas capitalisée)
        if ue_status["is_capitalized"] and ue_status["formsemestre_id"] is not None:
            sem_origin = FormSemestre.query.get(ue_status["formsemestre_id"])
            u[
                "ue_descr_txt"
            ] = f'capitalisée le {ndb.DateISOtoDMY(ue_status["event_date"])}'
            u["ue_descr_html"] = (
                f"""<a href="{ url_for( 'notes.formsemestre_bulletinetud',
            scodoc_dept=g.scodoc_dept, formsemestre_id=sem_origin.id, etudid=etudid)}"
            title="{sem_origin.titre_annee()}" class="bull_link"
            >{u["ue_descr_txt"]}</a>
            """
                if sem_origin
                else ""
            )
            if ue_status["moy"] != "NA":
                # détail des modules de l'UE capitalisée
                formsemestre_cap = FormSemestre.query.get(ue_status["formsemestre_id"])
                nt_cap: NotesTableCompat = res_sem.load_formsemestre_results(
                    formsemestre_cap
                )

                u["modules_capitalized"], _ = _ue_mod_bulletin(
                    etudid,
                    formsemestre_id,
                    ue_status["capitalized_ue_id"],
                    nt_cap.get_modimpls_dict(),
                    nt_cap,
                    version,
                )
                I["matieres_modules_capitalized"].update(
                    _sort_mod_by_matiere(u["modules_capitalized"], nt_cap, etudid)
                )
        else:
            if prefs["bul_show_ue_rangs"] and ue["type"] != codes_cursus.UE_SPORT:
                if ue_attente or nt.ue_rangs[ue["ue_id"]][0] is None:
                    u["ue_descr_txt"] = "%s/%s" % (
                        scu.RANG_ATTENTE_STR,
                        nt.ue_rangs[ue["ue_id"]][1],
                    )
                else:
                    u["ue_descr_txt"] = "%s/%s" % (
                        nt.ue_rangs[ue["ue_id"]][0][etudid],
                        nt.ue_rangs[ue["ue_id"]][1],
                    )
                u["ue_descr_html"] = u["ue_descr_txt"]
            else:
                u["ue_descr_txt"] = u["ue_descr_html"] = ""

        if ue_status["is_capitalized"] or modules:
            I["ues"].append(u)  # ne montre pas les UE si non inscrit

        # Accès par matieres
        # En #sco92, pas d'information
        I["matieres_modules"].update(_sort_mod_by_matiere(modules, nt, etudid))

    #
    C = make_context_dict(formsemestre, I["etud"])
    C.update(I)
    #
    # log( 'C = \n%s\n' % pprint.pformat(C) ) # tres pratique pour voir toutes les infos dispo
    return C


def get_appreciations_list(formsemestre_id: int, etudid: int) -> dict:
    """Appréciations pour cet étudiant dans ce semestre"""
    cnx = ndb.GetDBConnexion()
    apprecs = sco_etud.appreciations_list(
        cnx, args={"etudid": etudid, "formsemestre_id": formsemestre_id}
    )
    d = {
        "appreciations_list": apprecs,
        "appreciations_txt": [x["date"] + ": " + x["comment"] for x in apprecs],
    }
    # deprecated / keep it for backward compat in templates:
    d["appreciations"] = d["appreciations_txt"]
    return d


def _get_etud_etat_html(etat: str) -> str:
    """chaine html représentant l'état (backward compat sco7)"""
    if etat == scu.INSCRIT:
        return ""
    elif etat == scu.DEMISSION:
        return ' <font color="red">(DEMISSIONNAIRE)</font> '
    elif etat == scu.DEF:
        return ' <font color="red">(DEFAILLANT)</font> '
    else:
        return f' <font color="red">({etat})</font> '


def _sort_mod_by_matiere(modlist, nt, etudid):
    matmod = {}  # { matiere_id : [] }
    for mod in modlist:
        matiere_id = mod["module"]["matiere_id"]
        if matiere_id not in matmod:
            moy = nt.get_etud_mat_moy(matiere_id, etudid)
            matmod[matiere_id] = {
                "titre": mod["mat"]["titre"],
                "modules": mod,
                "moy": moy,
                "moy_txt": scu.fmt_note(moy),
            }
    return matmod


def _ue_mod_bulletin(
    etudid, formsemestre_id, ue_id, modimpls, nt: NotesTableCompat, version
):
    """Infos sur les modules (et évaluations) dans une UE
    (ajoute les informations aux modimpls)
    Result: liste de modules de l'UE avec les infos dans chacun (seulement ceux où l'étudiant est inscrit).
    """
    bul_show_mod_rangs = sco_preferences.get_preference(
        "bul_show_mod_rangs", formsemestre_id
    )
    bul_show_abs_modules = sco_preferences.get_preference(
        "bul_show_abs_modules", formsemestre_id
    )
    if bul_show_abs_modules:
        sem = sco_formsemestre.get_formsemestre(formsemestre_id)
        debut_sem = ndb.DateDMYtoISO(sem["date_debut"])
        fin_sem = ndb.DateDMYtoISO(sem["date_fin"])

    ue_modimpls = [mod for mod in modimpls if mod["module"]["ue_id"] == ue_id]
    mods = []  # result
    ue_attente = False  # true si une eval en attente dans cette UE
    for modimpl in ue_modimpls:
        mod_attente = False
        mod = modimpl.copy()
        mod_moy = nt.get_etud_mod_moy(
            modimpl["moduleimpl_id"], etudid
        )  # peut etre 'NI'
        is_malus = mod["module"]["module_type"] == ModuleType.MALUS
        if bul_show_abs_modules:
            nbabs, nbabsjust = sco_abs.get_abs_count(etudid, sem)
            mod_abs = [nbabs, nbabsjust]
            mod["mod_abs_txt"] = scu.fmt_abs(mod_abs)
        else:
            mod["mod_abs_txt"] = ""

        mod["mod_moy_txt"] = scu.fmt_note(mod_moy)
        if mod["mod_moy_txt"][:2] == "NA":
            mod["mod_moy_txt"] = "-"
        if is_malus:
            if isinstance(mod_moy, str):
                mod["mod_moy_txt"] = "-"
                mod["mod_coef_txt"] = "-"
            elif mod_moy > 0:
                mod["mod_moy_txt"] = scu.fmt_note(mod_moy)
                mod["mod_coef_txt"] = "Malus"
            elif mod_moy < 0:
                mod["mod_moy_txt"] = scu.fmt_note(-mod_moy)
                mod["mod_coef_txt"] = "Bonus"
            else:
                mod["mod_moy_txt"] = "-"
                mod["mod_coef_txt"] = "-"
        else:
            mod["mod_coef_txt"] = scu.fmt_coef(modimpl["module"]["coefficient"])
        if mod["mod_moy_txt"] != "NI":  # ne montre pas les modules 'non inscrit'
            mods.append(mod)
            if is_malus:  # n'affiche pas les statistiques sur les modules malus
                mod["stats"] = {
                    "moy": "",
                    "max": "",
                    "min": "",
                    "nb_notes": "",
                    "nb_missing": "",
                    "nb_valid_evals": "",
                }
            else:
                mod["stats"] = nt.get_mod_stats(modimpl["moduleimpl_id"])
            mod["mod_descr_txt"] = "Module %s, coef. %s (%s)" % (
                modimpl["module"]["titre"],
                scu.fmt_coef(modimpl["module"]["coefficient"]),
                sco_users.user_info(modimpl["responsable_id"])["nomcomplet"],
            )
            link_mod = (
                '<a class="bull_link" href="moduleimpl_status?moduleimpl_id=%s" title="%s">'
                % (modimpl["moduleimpl_id"], mod["mod_descr_txt"])
            )
            if sco_preferences.get_preference("bul_show_codemodules", formsemestre_id):
                mod["code"] = modimpl["module"]["code"]
                mod["code_html"] = link_mod + (mod["code"] or "") + "</a>"
            else:
                mod["code"] = mod["code_html"] = ""
            mod["name"] = (
                modimpl["module"]["abbrev"] or modimpl["module"]["titre"] or ""
            )
            mod["name_html"] = link_mod + mod["name"] + "</a>"

            mod_descr = "Module %s, coef. %s (%s)" % (
                modimpl["module"]["titre"],
                scu.fmt_coef(modimpl["module"]["coefficient"]),
                sco_users.user_info(modimpl["responsable_id"])["nomcomplet"],
            )
            link_mod = (
                '<a class="bull_link" href="moduleimpl_status?moduleimpl_id=%s" title="%s">'
                % (modimpl["moduleimpl_id"], mod_descr)
            )
            if sco_preferences.get_preference("bul_show_codemodules", formsemestre_id):
                mod["code_txt"] = modimpl["module"]["code"] or ""
                mod["code_html"] = link_mod + mod["code_txt"] + "</a>"
            else:
                mod["code_txt"] = ""
                mod["code_html"] = ""
            # Evaluations: notes de chaque eval
            evals = nt.get_evals_in_mod(modimpl["moduleimpl_id"])
            mod["evaluations"] = []
            for e in evals:
                e = e.copy()
                if e["visibulletin"] or version == "long":
                    # affiche "bonus" quand les points de malus sont négatifs
                    if is_malus:
                        val = e["notes"].get(etudid, {"value": "NP"})[
                            "value"
                        ]  # NA si etud demissionnaire
                        if val == "NP" or val > 0:
                            e["name"] = "Points de malus sur cette UE"
                        else:
                            e["name"] = "Points de bonus sur cette UE"
                    else:
                        e["name"] = e["description"] or f"le {e['jour']}"
                e["target_html"] = url_for(
                    "notes.evaluation_listenotes",
                    scodoc_dept=g.scodoc_dept,
                    evaluation_id=e["evaluation_id"],
                    format="html",
                    tf_submitted=1,
                )
                e[
                    "name_html"
                ] = f"""<a class="bull_link" href="{
                    e['target_html']}">{e['name']}</a>"""
                val = e["notes"].get(etudid, {"value": "NP"})["value"]
                # val est NP si etud demissionnaire
                if val == "NP":
                    e["note_txt"] = "nd"
                    e["note_html"] = '<span class="note_nd">nd</span>'
                    e["coef_txt"] = scu.fmt_coef(e["coefficient"])
                else:
                    # (-0.15) s'affiche "bonus de 0.15"
                    if is_malus:
                        val = abs(val)
                    e["note_txt"] = scu.fmt_note(val, note_max=e["note_max"])
                    e["note_html"] = e["note_txt"]
                    if is_malus:
                        e["coef_txt"] = ""
                    else:
                        e["coef_txt"] = scu.fmt_coef(e["coefficient"])
                if e["evaluation_type"] == scu.EVALUATION_RATTRAPAGE:
                    e["coef_txt"] = "rat."
                elif e["evaluation_type"] == scu.EVALUATION_SESSION2:
                    e["coef_txt"] = "Ses. 2"
                if e["etat"]["evalattente"]:
                    mod_attente = True  # une eval en attente dans ce module
                if ((not is_malus) or (val != "NP")) and (
                    (
                        e["evaluation_type"] == scu.EVALUATION_NORMALE
                        or not np.isnan(val)
                    )
                ):
                    # ne liste pas les eval malus sans notes
                    # ni les rattrapages et sessions 2 si pas de note
                    mod["evaluations"].append(e)

            # Evaluations incomplètes ou futures:
            mod["evaluations_incompletes"] = []
            if sco_preferences.get_preference("bul_show_all_evals", formsemestre_id):
                complete_eval_ids = set([e["evaluation_id"] for e in evals])
                all_evals = sco_evaluation_db.do_evaluation_list(
                    args={"moduleimpl_id": modimpl["moduleimpl_id"]}
                )
                all_evals.reverse()  # plus ancienne d'abord
                for e in all_evals:
                    if e["evaluation_id"] not in complete_eval_ids:
                        e = e.copy()
                        mod["evaluations_incompletes"].append(e)
                        e["name"] = (e["description"] or "") + " (%s)" % e["jour"]
                        e["target_html"] = url_for(
                            "notes.evaluation_listenotes",
                            scodoc_dept=g.scodoc_dept,
                            evaluation_id=e["evaluation_id"],
                            tf_submitted=1,
                            format="html",
                        )
                        e["name_html"] = '<a class="bull_link" href="%s">%s</a>' % (
                            e["target_html"],
                            e["name"],
                        )
                        e["note_txt"] = e["note_html"] = ""
                        e["coef_txt"] = scu.fmt_coef(e["coefficient"])
            # Classement
            if (
                bul_show_mod_rangs
                and (nt.mod_rangs is not None)
                and mod["mod_moy_txt"] != "-"
                and not is_malus
            ):
                rg = nt.mod_rangs[modimpl["moduleimpl_id"]]
                if rg[0] is None:
                    mod["mod_rang_txt"] = ""
                else:
                    if mod_attente:  # nt.get_moduleimpls_attente():
                        mod["mod_rang"] = scu.RANG_ATTENTE_STR
                    else:
                        mod["mod_rang"] = rg[0][etudid]
                    mod["mod_eff"] = rg[1]  # effectif dans ce module
                    mod["mod_rang_txt"] = "%s/%s" % (mod["mod_rang"], mod["mod_eff"])
            else:
                mod["mod_rang_txt"] = ""
        if mod_attente:
            ue_attente = True
    return mods, ue_attente


def get_etud_rangs_groups(
    etudid: int, partitions, partitions_etud_groups, nt: NotesTableCompat
):
    """Ramene rang et nb inscrits dans chaque partition"""
    rang_gr, ninscrits_gr, gr_name = {}, {}, {}
    for partition in partitions:
        if partition["partition_name"] != None:
            partition_id = partition["partition_id"]

            if etudid in partitions_etud_groups[partition_id]:
                group = partitions_etud_groups[partition_id][etudid]

                (
                    rang_gr[partition_id],
                    ninscrits_gr[partition_id],
                ) = nt.get_etud_rang_group(etudid, group["group_id"])
                gr_name[partition_id] = group["group_name"]
            else:  # etudiant non present dans cette partition
                rang_gr[partition_id], ninscrits_gr[partition_id] = "", ""
                gr_name[partition_id] = ""

    return rang_gr, ninscrits_gr, gr_name


def etud_descr_situation_semestre(
    etudid,
    formsemestre_id,
    ne="",
    format="html",  # currently unused
    show_decisions=True,
    show_uevalid=True,
    show_date_inscr=True,
    show_mention=False,
):
    """Dict décrivant la situation de l'étudiant dans ce semestre.
    Si format == 'html', peut inclure du balisage html (actuellement inutilisé)

    situation : chaine résumant en français la situation de l'étudiant.
                Par ex. "Inscrit le 31/12/1999. Décision jury: Validé. ..."

    date_inscription : (vide si show_date_inscr est faux)
    date_demission   : (vide si pas demission ou si show_date_inscr est faux)
    descr_inscription : "Inscrit" ou "Pas inscrit[e]"
    descr_demission   : "Démission le 01/02/2000" ou vide si pas de démission
    descr_defaillance  : "Défaillant" ou vide si non défaillant.
    decision_jury     :  "Validé", "Ajourné", ... (code semestre)
    descr_decision_jury : "Décision jury: Validé" (une phrase)
    decision_sem        :
    decisions_ue        : noms (acronymes) des UE validées, séparées par des virgules.
    descr_decisions_ue  : ' UE acquises: UE1, UE2', ou vide si pas de dec. ou si pas show_uevalid
    descr_mention : 'Mention Bien', ou vide si pas de mention ou si pas show_mention
    """
    # Fonction utilisée par tous les bulletins (APC ou classiques)
    cnx = ndb.GetDBConnexion()
    infos = scu.DictDefault(defaultvalue="")

    # --- Situation et décisions jury
    # démission/inscription ?
    events = sco_etud.scolar_events_list(
        cnx, args={"etudid": etudid, "formsemestre_id": formsemestre_id}
    )
    date_inscr = None
    date_dem = None
    date_def = None
    for event in events:
        event_type = event["event_type"]
        if event_type == "INSCRIPTION":
            if date_inscr:
                # plusieurs inscriptions ???
                # date_inscr += ', ' +   event['event_date'] + ' (!)'
                # il y a eu une erreur qui a laissé un event 'inscription'
                # on l'efface:
                log(
                    f"etud_descr_situation_semestre: removing duplicate INSCRIPTION event for etudid={etudid} !"
                )
                sco_etud.scolar_events_delete(cnx, event["event_id"])
            else:
                date_inscr = event["event_date"]
        elif event_type == "DEMISSION":
            # assert date_dem == None, 'plusieurs démissions !'
            if date_dem:  # cela ne peut pas arriver sauf bug (signale a Evry 2013?)
                log(
                    f"etud_descr_situation_semestre: removing duplicate DEMISSION event for etudid={etudid} !"
                )
                sco_etud.scolar_events_delete(cnx, event["event_id"])
            else:
                date_dem = event["event_date"]
        elif event_type == "DEFAILLANCE":
            if date_def:
                log(
                    f"etud_descr_situation_semestre: removing duplicate DEFAILLANCE event for etudid={etudid} !"
                )
                sco_etud.scolar_events_delete(cnx, event["event_id"])
            else:
                date_def = event["event_date"]
    if show_date_inscr:
        if not date_inscr:
            infos["date_inscription"] = ""
            infos["descr_inscription"] = f"Pas inscrit{ne}"
        else:
            infos["date_inscription"] = date_inscr
            infos["descr_inscription"] = f"Inscrit{ne} le {date_inscr}"
    else:
        infos["date_inscription"] = ""
        infos["descr_inscription"] = ""

    infos["descr_defaillance"] = ""

    # Décision: valeurs par defaut vides:
    infos["decision_jury"] = infos["descr_decision_jury"] = ""
    infos["decision_sem"] = ""
    infos["decisions_ue"] = infos["descr_decisions_ue"] = ""
    infos["descr_decisions_niveaux"] = infos["descr_decisions_rcue"] = ""
    infos["descr_decision_annee"] = ""

    if date_dem:
        infos["descr_demission"] = f"Démission le {date_dem}."
        infos["date_demission"] = date_dem
        infos["decision_jury"] = infos["descr_decision_jury"] = "Démission"
        infos["situation"] = ". ".join(
            [x for x in [infos["descr_inscription"], infos["descr_demission"]] if x]
        )
        return infos, None  # ne donne pas les dec. de jury pour les demissionnaires
    if date_def:
        infos["descr_defaillance"] = f"Défaillant{ne}"
        infos["date_defaillance"] = date_def
        infos["descr_decision_jury"] = f"Défaillant{ne}"

    dpv = sco_pvjury.dict_pvjury(formsemestre_id, etudids=[etudid])
    if dpv:
        infos["decision_sem"] = dpv["decisions"][0]["decision_sem"]

    if not show_decisions:
        return infos, dpv

    # Décisions de jury:
    pv = dpv["decisions"][0]
    descr_dec = ""
    if pv["decision_sem_descr"]:
        infos["decision_jury"] = pv["decision_sem_descr"]
        infos["descr_decision_jury"] = "Décision jury: " + pv["decision_sem_descr"]
        descr_dec = infos["descr_decision_jury"]
    else:
        infos["descr_decision_jury"] = ""
        infos["decision_jury"] = ""

    if pv["decisions_ue_descr"] and show_uevalid:
        infos["decisions_ue"] = pv["decisions_ue_descr"]
        infos["descr_decisions_ue"] = " UE acquises: " + pv["decisions_ue_descr"]
    else:
        infos["decisions_ue"] = ""
        infos["descr_decisions_ue"] = ""

    infos["mention"] = pv["mention"]
    if pv["mention"] and show_mention:
        descr_mention = f"Mention {pv['mention']}"
    else:
        descr_mention = ""

    # Décisions APC / BUT
    if pv.get("decision_annee", {}):
        infos["descr_decision_annee"] = "Décision année: " + pv.get(
            "decision_annee", {}
        ).get("code", "")
    else:
        infos["descr_decision_annee"] = ""

    infos["descr_decisions_rcue"] = pv.get("descr_decisions_rcue", "")
    infos["descr_decisions_niveaux"] = pv.get("descr_decisions_niveaux", "")

    descr_autorisations = ""
    if not pv["validation_parcours"]:  # parcours non terminé
        if pv["autorisations_descr"]:
            descr_autorisations = (
                f"Autorisé à s'inscrire en {pv['autorisations_descr']}."
            )
    else:
        descr_dec += " Diplôme obtenu."
    _format_situation_fields(
        infos,
        [
            "descr_inscription",
            "descr_defaillance",
            "descr_decisions_ue",
            "descr_decision_annee",
        ],
        [descr_dec, descr_mention, descr_autorisations],
    )

    return infos, dpv


def _format_situation_fields(
    infos, field_names: list[str], extra_values: list[str]
) -> None:
    """Réuni les champs pour former le paragraphe "situation", et ajoute la pontuation aux champs."""
    infos["situation"] = ". ".join(
        x
        for x in [infos.get(field_name, "") for field_name in field_names]
        + [field for field in extra_values if field]
        if x
    )
    for field_name in field_names:
        field = infos.get(field_name, "")
        if field and not field.endswith("."):
            infos[field_name] += "."


# ------ Page bulletin
def formsemestre_bulletinetud(
    etud: Identite = None,
    formsemestre_id=None,
    format=None,
    version="long",
    xml_with_decisions=False,
    force_publishing=False,  # force publication meme si semestre non publie sur "portail"
    prefer_mail_perso=False,
):
    """Page bulletin de notes pour
    - HTML des formations classiques (non BUT)
    - le format "oldjson" (les "json" sont générés à part, voir get_formsemestre_bulletin_etud_json)
    - les formats PDF, XML et mail pdf (toutes formations)

    Note: le format XML n'est plus maintenu et pour les BUT ne contient pas
    toutes les informations. Privilégier le format JSON.

    Paramètres:
    - version: pour les formations classqiues, versions short/selectedevals/long
    - xml_with_decisions: inclue ou non les
    - force_publishing: renvoie le bulletin même si semestre non publie sur "portail"
    - prefer_mail_perso: pour pdfmail, utilise adresse mail perso en priorité.

    """
    format = format or "html"
    formsemestre: FormSemestre = FormSemestre.query.get(formsemestre_id)
    if not formsemestre:
        raise ScoValueError(f"semestre {formsemestre_id} inconnu !")

    bulletin = do_formsemestre_bulletinetud(
        formsemestre,
        etud.id,
        format=format,
        version=version,
        xml_with_decisions=xml_with_decisions,
        force_publishing=force_publishing,
        prefer_mail_perso=prefer_mail_perso,
    )[0]
    if format not in {"html", "pdfmail"}:
        filename = scu.bul_filename(formsemestre, etud, format)
        return scu.send_file(bulletin, filename, mime=scu.get_mime_suffix(format)[0])
    elif format == "pdfmail":
        return ""
    H = [
        _formsemestre_bulletinetud_header_html(etud, formsemestre, format, version),
        bulletin,
        render_template(
            "bul_foot.j2",
            appreciations=None,  # déjà affichées
            css_class="bul_classic_foot",
            etud=etud,
            formsemestre=formsemestre,
            inscription_courante=etud.inscription_courante(),
            inscription_str=etud.inscription_descr()["inscription_str"],
        ),
        html_sco_header.sco_footer(),
    ]

    return "".join(H)


def can_send_bulletin_by_mail(formsemestre_id):
    """True if current user is allowed to send a bulletin by mail"""
    sem = sco_formsemestre.get_formsemestre(formsemestre_id)
    return (
        sco_preferences.get_preference("bul_mail_allowed_for_all", formsemestre_id)
        or current_user.has_permission(Permission.ScoImplement)
        or current_user.id in sem["responsables"]
    )


def do_formsemestre_bulletinetud(
    formsemestre: FormSemestre,
    etudid: int,
    version="long",  # short, long, selectedevals
    format=None,
    xml_with_decisions: bool = False,
    force_publishing: bool = False,
    prefer_mail_perso: bool = False,
    with_img_signatures_pdf: bool = True,
):
    """Génère le bulletin au format demandé.
    Utilisé pour:
    - HTML des formations classiques (non BUT)
    - le format "oldjson" (les json sont générés à part, voir get_formsemestre_bulletin_etud_json)
    - les formats PDF, XML et mail pdf (toutes formations)

    Options:
    - xml_with_decisions:      force décisions dans XML
    - force_publishing:        force publication meme si semestre non publié sur "portail"
    - prefer_mail_perso:       mails envoyés sur adresse perso si non vide
    - with_img_signatures_pdf: si faux, ne met pas les signatures dans le footer PDF.

    Résultat: (bul, filigranne)
    où bul est str ou bytes au format demandé (html, pdf, pdfmail, pdfpart, xml, json)
    et filigranne est un message à placer en "filigranne" (eg "Provisoire").
    """
    format = format or "html"
    if format == "xml":
        bul = sco_bulletins_xml.make_xml_formsemestre_bulletinetud(
            formsemestre.id,
            etudid,
            xml_with_decisions=xml_with_decisions,
            force_publishing=force_publishing,
            version=version,
        )

        return bul, ""

    elif format == "json":  # utilisé pour classic et "oldjson"
        bul = sco_bulletins_json.make_json_formsemestre_bulletinetud(
            formsemestre.id,
            etudid,
            xml_with_decisions=xml_with_decisions,
            force_publishing=force_publishing,
            version=version,
        )
        return bul, ""
    if version.endswith("_mat"):
        version = version[:-4]  # enlève le "_mat"

    if formsemestre.formation.is_apc():
        etudiant = Identite.query.get(etudid)
        r = bulletin_but.BulletinBUT(formsemestre)
        infos = r.bulletin_etud_complet(etudiant, version=version)
    else:
        infos = formsemestre_bulletinetud_dict(formsemestre.id, etudid)
    etud = infos["etud"]

    if format == "html":
        htm, _ = sco_bulletins_generator.make_formsemestre_bulletinetud(
            infos, version=version, format="html"
        )
        return htm, infos["filigranne"]

    elif format == "pdf" or format == "pdfpart":
        bul, filename = sco_bulletins_generator.make_formsemestre_bulletinetud(
            infos,
            version=version,
            format="pdf",
            stand_alone=(format != "pdfpart"),
            with_img_signatures_pdf=with_img_signatures_pdf,
        )
        if format == "pdf":
            return (
                scu.sendPDFFile(bul, filename),
                infos["filigranne"],
            )  # unused ret. value
        else:
            return bul, infos["filigranne"]

    elif format == "pdfmail":
        # format pdfmail: envoie le pdf par mail a l'etud, et affiche le html
        # check permission
        if not can_send_bulletin_by_mail(formsemestre.id):
            raise AccessDenied("Vous n'avez pas le droit d'effectuer cette opération !")

        pdfdata, filename = sco_bulletins_generator.make_formsemestre_bulletinetud(
            infos, version=version, format="pdf"
        )

        if prefer_mail_perso:
            recipient_addr = etud.get("emailperso", "") or etud.get("email", "")
        else:
            recipient_addr = etud.get("email", "") or etud.get("emailperso", "")

        if not recipient_addr:
            flash(f"{etud['nomprenom']} n'a pas d'adresse e-mail !")
            return False, infos["filigranne"]
        else:
            mail_bulletin(formsemestre.id, infos, pdfdata, filename, recipient_addr)
            flash(f"mail envoyé à {recipient_addr}")

            return True, infos["filigranne"]

    raise ValueError("do_formsemestre_bulletinetud: invalid format (%s)" % format)


def mail_bulletin(formsemestre_id, infos, pdfdata, filename, recipient_addr):
    """Send bulletin by email to etud
    If bul_mail_list_abs pref is true, put list of absences in mail body (text).
    """
    etud = infos["etud"]
    webmaster = sco_preferences.get_preference("bul_mail_contact_addr", formsemestre_id)
    dept = scu.unescape_html(
        sco_preferences.get_preference("DeptName", formsemestre_id)
    )
    copy_addr = sco_preferences.get_preference("email_copy_bulletins", formsemestre_id)
    intro_mail = sco_preferences.get_preference("bul_intro_mail", formsemestre_id)

    if intro_mail:
        try:
            hea = intro_mail % {
                "nomprenom": etud["nomprenom"],
                "dept": dept,
                "webmaster": webmaster,
            }
        except KeyError as e:
            raise ScoValueError(
                "format 'Message d'accompagnement' (bul_intro_mail) invalide, revoir les réglages dans les préférences"
            ) from e
    else:
        hea = ""

    if sco_preferences.get_preference("bul_mail_list_abs"):
        hea += "\n\n" + sco_abs_views.ListeAbsEtud(
            etud["etudid"], with_evals=False, format="text"
        )

    subject = f"""Relevé de notes de {etud["nomprenom"]}"""
    recipients = [recipient_addr]
    sender = sco_preferences.get_preference("email_from_addr", formsemestre_id)
    if copy_addr:
        bcc = copy_addr.strip().split(",")
    else:
        bcc = ""

    # Attach pdf
    log(f"""mail bulletin a {recipient_addr}""")
    email.send_email(
        subject,
        sender,
        recipients,
        bcc=bcc,
        text_body=hea,
        attachments=[
            {"filename": filename, "mimetype": scu.PDF_MIMETYPE, "data": pdfdata}
        ],
    )


def make_menu_autres_operations(
    formsemestre: FormSemestre, etud: Identite, endpoint: str, version: str
) -> str:
    etud_email = etud.get_first_email() or ""
    etud_perso = etud.get_first_email("emailperso") or ""
    menu_items = [
        {
            "title": "Réglages bulletins",
            "endpoint": "notes.formsemestre_edit_options",
            "args": {
                "formsemestre_id": formsemestre.id,
                # "target_url": url_for(
                #     "notes.formsemestre_bulletinetud",
                #     scodoc_dept=g.scodoc_dept,
                #     formsemestre_id=formsemestre_id,
                #     etudid=etudid,
                # ),
            },
            "enabled": formsemestre.can_be_edited_by(current_user),
        },
        {
            "title": 'Version papier (pdf, format "%s")'
            % sco_bulletins_generator.bulletin_get_class_name_displayed(
                formsemestre.id
            ),
            "endpoint": endpoint,
            "args": {
                "formsemestre_id": formsemestre.id,
                "etudid": etud.id,
                "version": version,
                "format": "pdf",
            },
        },
        {
            "title": f"Envoi par mail à {etud_email}",
            "endpoint": endpoint,
            "args": {
                "formsemestre_id": formsemestre.id,
                "etudid": etud.id,
                "version": version,
                "format": "pdfmail",
            },
            # possible slt si on a un mail...
            "enabled": etud_email and can_send_bulletin_by_mail(formsemestre.id),
        },
        {
            "title": f"Envoi par mail à {etud_perso} (adr. personnelle)",
            "endpoint": endpoint,
            "args": {
                "formsemestre_id": formsemestre.id,
                "etudid": etud.id,
                "version": version,
                "format": "pdfmail",
                "prefer_mail_perso": 1,
            },
            # possible slt si on a un mail...
            "enabled": etud_perso and can_send_bulletin_by_mail(formsemestre.id),
        },
        {
            "title": "Version json",
            "endpoint": endpoint,
            "args": {
                "formsemestre_id": formsemestre.id,
                "etudid": etud.id,
                "version": version,
                "format": "json",
            },
        },
        {
            "title": "Version XML",
            "endpoint": endpoint,
            "args": {
                "formsemestre_id": formsemestre.id,
                "etudid": etud.id,
                "version": version,
                "format": "xml",
            },
        },
        {
            "title": "Ajouter une appréciation",
            "endpoint": "notes.appreciation_add_form",
            "args": {
                "formsemestre_id": formsemestre.id,
                "etudid": etud.id,
            },
            "enabled": (
                formsemestre.can_be_edited_by(current_user)
                or current_user.has_permission(Permission.ScoEtudInscrit)
            ),
        },
        {
            "title": "Enregistrer un semestre effectué ailleurs",
            "endpoint": "notes.formsemestre_ext_create_form",
            "args": {
                "formsemestre_id": formsemestre.id,
                "etudid": etud.id,
            },
            "enabled": current_user.has_permission(Permission.ScoImplement),
        },
        {
            "title": "Enregistrer une validation d'UE antérieure",
            "endpoint": "notes.formsemestre_validate_previous_ue",
            "args": {
                "formsemestre_id": formsemestre.id,
                "etudid": etud.id,
            },
            "enabled": formsemestre.can_edit_jury(),
        },
        {
            "title": "Enregistrer note d'une UE externe",
            "endpoint": "notes.external_ue_create_form",
            "args": {
                "formsemestre_id": formsemestre.id,
                "etudid": etud.id,
            },
            "enabled": formsemestre.can_edit_jury()
            and not formsemestre.formation.is_apc(),
        },
        {
            "title": "Entrer décisions jury",
            "endpoint": "notes.formsemestre_validation_etud_form",
            "args": {
                "formsemestre_id": formsemestre.id,
                "etudid": etud.id,
            },
            "enabled": formsemestre.can_edit_jury(),
        },
        {
            "title": "Éditer PV jury",
            "endpoint": "notes.formsemestre_pvjury_pdf",
            "args": {
                "formsemestre_id": formsemestre.id,
                "etudid": etud.id,
            },
            "enabled": True,
        },
    ]
    return htmlutils.make_menu("Autres opérations", menu_items, alone=True)


def _formsemestre_bulletinetud_header_html(
    etud,
    formsemestre: FormSemestre,
    format=None,
    version=None,
):
    H = [
        html_sco_header.sco_header(
            page_title=f"Bulletin de {etud.nomprenom}",
            javascripts=[
                "js/bulletin.js",
                "libjs/d3.v3.min.js",
                "js/radar_bulletin.js",
            ],
            cssstyles=["css/radar_bulletin.css"],
        ),
        render_template(
            "bul_head.j2",
            etud=etud,
            format=format,
            formsemestre=formsemestre,
            menu_autres_operations=make_menu_autres_operations(
                etud=etud,
                formsemestre=formsemestre,
                endpoint="notes.formsemestre_bulletinetud",
                version=version,
            ),
            scu=scu,
            time=time,
            version=version,
        ),
    ]
    return "\n".join(H)