ScoDoc/app/scodoc/sco_utils.py

# -*- mode: python -*-
# -*- coding: utf-8 -*-

##############################################################################
#
# Gestion scolarite IUT
#
# Copyright (c) 1999 - 2023 Emmanuel Viennet.  All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#   Emmanuel Viennet      emmanuel.viennet@viennet.net
#
##############################################################################


""" Common definitions
"""
import base64
import bisect
import collections
import datetime
from enum import IntEnum, Enum
import io
import json
from hashlib import md5
import numbers
import os
import re
from shutil import get_terminal_size
import _thread
import time
import unicodedata
import urllib
from urllib.parse import urlparse, parse_qsl, urlunparse, urlencode

import numpy as np
from PIL import Image as PILImage
import pydot
import requests

from pytz import timezone

import dateutil.parser as dtparser

import flask
from flask import g, request, Response
from flask import flash, url_for, make_response
from flask_json import json_response
from werkzeug.http import HTTP_STATUS_CODES

from config import Config
from app import log, ScoDocJSONEncoder

from app.scodoc.codes_cursus import NOTES_TOLERANCE, CODES_EXPL
from app.scodoc import sco_xml
import sco_version

# En principe, aucun champ text ne devrait excéder cette taille
MAX_TEXT_LEN = 64 * 1024

# le répertoire static, lié à chaque release pour éviter les problèmes de caches
STATIC_DIR = (
    os.environ.get("SCRIPT_NAME", "") + "/ScoDoc/static/links/" + sco_version.SCOVERSION
)
# La time zone du serveur:
TIME_ZONE = timezone("/".join(os.path.realpath("/etc/localtime").split("/")[-2:]))

# ----- CALCUL ET PRESENTATION DES NOTES
NOTES_PRECISION = 1e-4  # evite eventuelles erreurs d'arrondis
NOTES_MIN = 0.0  # valeur minimale admise pour une note (sauf malus, dans [-20, 20])
NOTES_MAX = 1000.0
NOTES_ABSENCE = -999.0  # absences dans les DataFrames, NULL en base
NOTES_NEUTRALISE = -1000.0  # notes non prises en comptes dans moyennes
NOTES_SUPPRESS = -1001.0  # note a supprimer
NOTES_ATTENTE = -1002.0  # note "en attente" (se calcule comme une note neutralisee)

NO_NOTE_STR = "-"  # contenu des cellules de tableaux html sans notes

# ---- CODES INSCRIPTION AUX SEMESTRES
# (champ etat de FormSemestreInscription)
INSCRIT = "I"
DEMISSION = "D"
DEF = "DEF"
ETATS_INSCRIPTION = {
    INSCRIT: "Inscrit",
    DEMISSION: "Démission",
    DEF: "Défaillant",
}


def print_progress_bar(
    iteration,
    total,
    prefix="",
    suffix="",
    finish_msg="",
    decimals=1,
    length=100,
    fill="█",
    autosize=False,
):
    """
    Affiche une progress bar à un point donné (mettre dans une boucle pour rendre dynamique)
    @params:
        iteration   - Required  : index du point donné (Int)
        total       - Required  : nombre total avant complétion (eg: len(List))
        prefix      - Optional  : Préfix -> écrit à gauche de la barre (Str)
        suffix      - Optional  : Suffix -> écrit à droite de la barre (Str)
        decimals    - Optional  : nombres de chiffres après la virgule (Int)
        length      - Optional  : taille de la barre en nombre de caractères (Int)
        fill        - Optional  : charactère de remplissange de la barre (Str)
        autosize    - Optional  : Choisir automatiquement la taille de la barre en fonction du terminal (Bool)
    """
    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
    color = TerminalColor.RED
    if 50 >= float(percent) > 25:
        color = TerminalColor.MAGENTA
    if 75 >= float(percent) > 50:
        color = TerminalColor.BLUE
    if 90 >= float(percent) > 75:
        color = TerminalColor.CYAN
    if 100 >= float(percent) > 90:
        color = TerminalColor.GREEN
    styling = f"{prefix} |{fill}| {percent}% {suffix}"
    if autosize:
        cols, _ = get_terminal_size(fallback=(length, 1))
        length = cols - len(styling)
    filled_length = int(length * iteration // total)
    pg_bar = fill * filled_length + "-" * (length - filled_length)
    print(f"\r{color}{styling.replace(fill, pg_bar)}{TerminalColor.RESET}", end="\r")
    # Affiche une nouvelle ligne vide
    if iteration == total:
        print(f"\n{finish_msg}")


class TerminalColor:
    """Ensemble de couleur pour terminaux"""

    BLUE = "\033[94m"
    CYAN = "\033[96m"
    GREEN = "\033[92m"
    MAGENTA = "\033[95m"
    RED = "\033[91m"
    RESET = "\033[0m"


class BiDirectionalEnum(Enum):
    """Permet la recherche inverse d'un enum
    Condition : les clés et les valeurs doivent être uniques
                les clés doivent être en MAJUSCULES
    """

    @classmethod
    def contains(cls, attr: str):
        """Vérifie sur un attribut existe dans l'enum"""
        return attr.upper() in cls._member_names_

    @classmethod
    def all(cls, keys=True):
        """Retourne toutes les clés de l'enum"""
        return cls._member_names_ if keys else list(cls._value2member_map_.keys())

    @classmethod
    def get(cls, attr: str, default: any = None):
        """Récupère une valeur à partir de son attribut"""
        val = None
        try:
            val = cls[attr.upper()]
        except (KeyError, AttributeError):
            val = default
        return val

    @classmethod
    def inverse(cls):
        """Retourne un dictionnaire représentant la map inverse de l'Enum"""
        return cls._value2member_map_


class EtatAssiduite(int, BiDirectionalEnum):
    """Code des états d'assiduité"""

    # Stockés en BD ne pas modifier

    PRESENT = 0
    RETARD = 1
    ABSENT = 2


class EtatJustificatif(int, BiDirectionalEnum):
    """Code des états des justificatifs"""

    # Stockés en BD ne pas modifier

    VALIDE = 0
    NON_VALIDE = 1
    ATTENTE = 2
    MODIFIE = 3


def is_iso_formated(date: str, convert=False) -> bool or datetime.datetime or None:
    """
    Vérifie si une date est au format iso

    Retourne un booléen Vrai (ou un objet Datetime si convert = True)
    si l'objet est au format iso

    Retourne Faux si l'objet n'est pas au format et convert = False

    Retourne None sinon
    """

    try:
        date: datetime.datetime = dtparser.isoparse(date)
        return date if convert else True
    except (dtparser.ParserError, ValueError, TypeError):
        return None if convert else False


def localize_datetime(date: datetime.datetime or str) -> datetime.datetime:
    """Ajoute un timecode UTC à la date donnée."""
    if isinstance(date, str):
        date = is_iso_formated(date, convert=True)

    new_date: datetime.datetime = date
    if new_date.tzinfo is None:
        try:
            new_date = TIME_ZONE.localize(date)
        except OverflowError:
            new_date = timezone("UTC").localize(date)
    return new_date


def is_period_overlapping(
    periode: tuple[datetime.datetime, datetime.datetime],
    interval: tuple[datetime.datetime, datetime.datetime],
    bornes: bool = True,
) -> bool:
    """
    Vérifie si la période et l'interval s'intersectent
    si strict == True : les extrémitées ne comptes pas
    Retourne Vrai si c'est le cas, faux sinon
    """
    p_deb, p_fin = periode
    i_deb, i_fin = interval

    if bornes:
        return p_deb <= i_fin and p_fin >= i_deb
    return p_deb < i_fin and p_fin > i_deb


class AssiduitesMetrics:
    """Labels associés au métrique de l'assiduité"""

    SHORT: list[str] = ["1/2 J.", "J.", "H."]
    LONG: list[str] = ["Demi-journée", "Journée", "Heure"]
    TAG: list[str] = ["demi", "journee", "heure"]


def translate_assiduites_metric(metric, inverse=True, short=True) -> str:
    """
    translate_assiduites_metric

    SHORT[true] : "J." "H." "N." "1/2 J."
    SHORT[false] : "Journée" "Heure" "Nombre" "Demi-Journée"

    inverse[false] : "demi" -> "1/2 J."
    inverse[true] : "1/2 J." -> "demi"


    Args:
        metric (str): la métrique à traduire
        inverse (bool, optional). Defaults to True.
        short (bool, optional). Defaults to True.

    Returns:
        str: la métrique traduite
    """
    index: int = None
    if not inverse:
        try:
            index = AssiduitesMetrics.TAG.index(metric)
            return (
                AssiduitesMetrics.SHORT[index]
                if short
                else AssiduitesMetrics.LONG[index]
            )
        except ValueError:
            return None

    try:
        index = (
            AssiduitesMetrics.SHORT.index(metric)
            if short
            else AssiduitesMetrics.LONG.index(metric)
        )
        return AssiduitesMetrics.TAG[index]
    except ValueError:
        return None


# Types de modules
class ModuleType(IntEnum):
    """Code des types de module."""

    # Stockés en BD dans Module.module_type: ne pas modifier ces valeurs
    STANDARD = 0
    MALUS = 1
    RESSOURCE = 2  # BUT
    SAE = 3  # BUT

    @classmethod
    def get_abbrev(cls, code) -> str:
        """Abbréviation décrivant le type de module à partir du code integer:
        "mod", "malus", "res", "sae"
        (utilisées pour style CSS)
        """
        return {
            ModuleType.STANDARD: "mod",
            ModuleType.MALUS: "malus",
            ModuleType.RESSOURCE: "res",
            ModuleType.SAE: "sae",
        }.get(code, "???")


MODULE_TYPE_NAMES = {
    ModuleType.STANDARD: "Module",
    ModuleType.MALUS: "Malus",
    ModuleType.RESSOURCE: "Ressource",
    ModuleType.SAE: "SAÉ",
    None: "Module",
}

PARTITION_PARCOURS = "Parcours"

MALUS_MAX = 20.0
MALUS_MIN = -20.0

APO_MISSING_CODE_STR = "----"  # shown in HTML pages in place of missing code Apogée
EDIT_NB_ETAPES = 6  # Nombre max de codes étapes / semestre presentés dans l'UI

IT_SITUATION_MISSING_STR = (
    "____"  # shown on ficheEtud (devenir) in place of empty situation
)

RANG_ATTENTE_STR = "(attente)"  #  rang affiché sur bulletins quand notes en attente

# borne supérieure de chaque mention
NOTES_MENTIONS_TH = (
    NOTES_TOLERANCE,
    7.0,
    10.0,
    12.0,
    14.0,
    16.0,
    18.0,
    20.0 + NOTES_TOLERANCE,
)
NOTES_MENTIONS_LABS = (
    "Nul",
    "Faible",
    "Insuffisant",
    "Passable",
    "Assez bien",
    "Bien",
    "Très bien",
    "Excellent",
)

EVALUATION_NORMALE = 0
EVALUATION_RATTRAPAGE = 1
EVALUATION_SESSION2 = 2

# Dates et années scolaires
# Ces dates "pivot" sont paramétrables dans les préférences générales
# on donne ici les valeurs par défaut.
# Les semestres commençant à partir du 1er août 20XX sont
# dans l'année scolaire 20XX
MONTH_DEBUT_ANNEE_SCOLAIRE = 8  # août
# Les semestres commençant à partir du 1er décembre
# sont "2eme période" (S_pair):
MONTH_DEBUT_PERIODE2 = MONTH_DEBUT_ANNEE_SCOLAIRE + 4

MONTH_NAMES_ABBREV = (
    "Jan ",
    "Fév ",
    "Mars",
    "Avr ",
    "Mai ",
    "Juin",
    "Jul ",
    "Août",
    "Sept",
    "Oct ",
    "Nov ",
    "Déc ",
)

MONTH_NAMES = (
    "janvier",
    "février",
    "mars",
    "avril",
    "mai",
    "juin",
    "juillet",
    "août",
    "septembre",
    "octobre",
    "novembre",
    "décembre",
)
DAY_NAMES = ("lundi", "mardi", "mercredi", "jeudi", "vendredi", "samedi", "dimanche")


def fmt_note(val, note_max=None, keep_numeric=False):
    """conversion note en str pour affichage dans tables HTML ou PDF.
    Si keep_numeric, laisse les valeur numeriques telles quelles (pour export Excel)
    """
    if val is None or val == NOTES_ABSENCE:
        return "ABS"
    if val == NOTES_NEUTRALISE:
        return "EXC"  # excuse, note neutralise
    if val == NOTES_ATTENTE:
        return "ATT"  # attente, note neutralisee
    if not isinstance(val, str):
        if np.isnan(val):
            return "~"
        if (note_max is not None) and note_max > 0:
            val = val * 20.0 / note_max
        if keep_numeric:
            return val
        else:
            s = "%2.2f" % round(float(val), 2)  # 2 chiffres apres la virgule
            s = "0" * (5 - len(s)) + s  # padding: 0 à gauche pour longueur 5: "12.34"
            return s
    else:
        return val.replace("NA", "-")


def fmt_coef(val):
    """Conversion valeur coefficient (float) en chaine"""
    if val < 0.01:
        return "%g" % val  # unusually small value
    return "%g" % round(val, 2)


def fmt_abs(val):
    """Conversion absences en chaine. val est une list [nb_abs_total, nb_abs_justifiees
    => NbAbs / Nb_justifiees
    """
    return "%s / %s" % (val[0], val[1])


def isnumber(x):
    "True if x is a number (int, float, etc.)"
    return isinstance(x, numbers.Number)


def jsnan(x):
    "if x is NaN, returns None"
    if isinstance(x, numbers.Number) and np.isnan(x):
        return None
    return x


def join_words(*words):
    words = [str(w).strip() for w in words if w is not None]
    return " ".join([w for w in words if w])


def get_mention(moy):
    """Texte "mention" en fonction de la moyenne générale"""
    try:
        moy = float(moy)
    except:
        return ""
    if moy > 0.0:
        return NOTES_MENTIONS_LABS[bisect.bisect_right(NOTES_MENTIONS_TH, moy)]
    else:
        return ""


def group_by_key(d: dict, key) -> dict:
    grouped = collections.defaultdict(lambda: [])
    for e in d:
        grouped[e[key]].append(e)
    return grouped


# ----- Global lock for critical sections (except notes_tables caches)
GSL = _thread.allocate_lock()  # Global ScoDoc Lock

SCODOC_DIR = Config.SCODOC_DIR

# ----- Repertoire "config" modifiable
#        /opt/scodoc-data/config
SCODOC_CFG_DIR = os.path.join(Config.SCODOC_VAR_DIR, "config")
# ----- Version information
SCODOC_VERSION_DIR = os.path.join(SCODOC_CFG_DIR, "version")
# ----- Repertoire tmp : /opt/scodoc-data/tmp
SCO_TMP_DIR = os.path.join(Config.SCODOC_VAR_DIR, "tmp")
if not os.path.exists(SCO_TMP_DIR) and os.path.exists(Config.SCODOC_VAR_DIR):
    os.mkdir(SCO_TMP_DIR, 0o755)
# ----- Les logos: /opt/scodoc-data/config/logos
SCODOC_LOGOS_DIR = os.path.join(SCODOC_CFG_DIR, "logos")
LOGOS_IMAGES_ALLOWED_TYPES = ("jpg", "jpeg", "png")  # remind that PIL does not read pdf
LOGOS_DIR_PREFIX = "logos_"
LOGO_FILE_PREFIX = "logo_"

# forme générale des noms des fichiers logos/background:
# SCODOC_LOGO_DIR/LOGO_FILE_PREFIX<name>.<suffix>   (fichier global) ou
# SCODOC_LOGO_DIR/LOGOS_DIR_PREFIX<dept_id>/LOGO_FILE_PREFIX<name>.<suffix> (fichier départemental)

# ----- Les outils distribués
SCO_TOOLS_DIR = os.path.join(Config.SCODOC_DIR, "tools")


# ----- Lecture du fichier de configuration
from app.scodoc import sco_config
from app.scodoc import sco_config_load

sco_config_load.load_local_configuration(SCODOC_CFG_DIR)
CONFIG = sco_config.CONFIG
if hasattr(CONFIG, "CODES_EXPL"):
    CODES_EXPL.update(
        CONFIG.CODES_EXPL
    )  # permet de customiser les explications de codes

if CONFIG.CUSTOM_HTML_HEADER:
    CUSTOM_HTML_HEADER = open(CONFIG.CUSTOM_HTML_HEADER).read()
else:
    CUSTOM_HTML_HEADER = ""

if CONFIG.CUSTOM_HTML_HEADER_CNX:
    CUSTOM_HTML_HEADER_CNX = open(CONFIG.CUSTOM_HTML_HEADER_CNX).read()
else:
    CUSTOM_HTML_HEADER_CNX = ""

if CONFIG.CUSTOM_HTML_FOOTER:
    CUSTOM_HTML_FOOTER = open(CONFIG.CUSTOM_HTML_FOOTER).read()
else:
    CUSTOM_HTML_FOOTER = ""

if CONFIG.CUSTOM_HTML_FOOTER_CNX:
    CUSTOM_HTML_FOOTER_CNX = open(CONFIG.CUSTOM_HTML_FOOTER_CNX).read()
else:
    CUSTOM_HTML_FOOTER_CNX = ""

SCO_ENCODING = "utf-8"  # used by Excel, XML, PDF, ...


SCO_DEFAULT_SQL_USER = "scodoc"  # should match Zope process UID
SCO_DEFAULT_SQL_PORT = "5432"
SCO_DEFAULT_SQL_USERS_CNX = "dbname=SCOUSERS port=%s" % SCO_DEFAULT_SQL_PORT

# Valeurs utilisées pour affichage seulement, pas de requetes ni de mails envoyés:
SCO_WEBSITE = "https://scodoc.org"
SCO_USER_MANUAL = "https://scodoc.org/GuideUtilisateur"
SCO_ANNONCES_WEBSITE = "https://scodoc.org/Contact"
SCO_DEVEL_LIST = "scodoc-devel@listes.univ-paris13.fr"
SCO_USERS_LIST = "notes@listes.univ-paris13.fr"
SCO_LISTS_URL = "https://scodoc.org/Contact"
SCO_DISCORD_ASSISTANCE = "https://discord.gg/ybw6ugtFsZ"

# Mails avec exceptions (erreurs) anormales envoyés à cette adresse:
# mettre '' pour désactiver completement l'envois de mails d'erreurs.
# (ces mails sont précieux pour corriger les erreurs, ne les désactiver que si
#  vous avez de bonnes raisons de le faire: vous pouvez me contacter avant)
SCO_EXC_MAIL = "scodoc-exception@viennet.net"

# L'adresse du mainteneur (non utilisée automatiquement par ScoDoc: ne pas changer)
SCO_DEV_MAIL = "emmanuel.viennet@gmail.com"  # SVP ne pas changer

# Adresse pour l'envoi des dumps (pour assistance technnique):
#   ne pas changer (ou vous perdez le support)
SCO_DUMP_UP_URL = "https://scodoc.org/scodoc-installmgr/upload-dump"
SCO_UP2DATE = "https://scodoc.org/scodoc-installmgr/check_version"
SCO_ORG_TIMEOUT = 180  # contacts scodoc.org
SCO_EXT_TIMEOUT = 180  # appels à des ressources extérieures (siret, ...)
SCO_TEST_API_TIMEOUT = 5  # pour tests unitaires API
CSV_FIELDSEP = ";"
CSV_LINESEP = "\n"
CSV_MIMETYPE = "text/comma-separated-values"
CSV_SUFFIX = ".csv"
DOCX_MIMETYPE = (
    "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
)
DOCX_SUFFIX = ".docx"
JSON_MIMETYPE = "application/json"
JSON_SUFFIX = ".json"
PDF_MIMETYPE = "application/pdf"
PDF_SUFFIX = ".pdf"
XLS_MIMETYPE = "application/vnd.ms-excel"
XLS_SUFFIX = ".xls"
XLSX_MIMETYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
XLSX_SUFFIX = ".xlsx"
XML_MIMETYPE = "text/xml"
XML_SUFFIX = ".xml"

# Format pour lesquels on exporte sans formattage des nombres (pas de perte de précision)
FORMATS_NUMERIQUES = {"csv", "xls", "xlsx", "xml", "json"}


def get_mime_suffix(format_code: str) -> tuple[str, str]:
    """Returns (MIME, SUFFIX) from format_code == "xls", "xml", ...
    SUFFIX includes the dot: ".xlsx", ".xml", ...
    "xls" and "xlsx" format codes give XLSX
    """
    d = {
        "csv": (CSV_MIMETYPE, CSV_SUFFIX),
        "docx": (DOCX_MIMETYPE, DOCX_SUFFIX),
        "xls": (XLSX_MIMETYPE, XLSX_SUFFIX),
        "xlsx": (XLSX_MIMETYPE, XLSX_SUFFIX),
        "pdf": (PDF_MIMETYPE, PDF_SUFFIX),
        "xml": (XML_MIMETYPE, XML_SUFFIX),
        "json": (JSON_MIMETYPE, JSON_SUFFIX),
    }
    return d[format_code]


# Admissions des étudiants
# Différents types de voies d'admission:
# (stocké en texte libre dans la base, mais saisie par menus pour harmoniser)
TYPE_ADMISSION_DEFAULT = "Inconnue"
TYPES_ADMISSION = (TYPE_ADMISSION_DEFAULT, "APB", "APB-PC", "CEF", "Direct")

BULLETINS_VERSIONS = ("short", "selectedevals", "long")

# Support for ScoDoc7 compatibility


def ScoURL():
    """base URL for this sco instance.
    e.g. https://scodoc.xxx.fr/ScoDoc/DEPT/Scolarite
    = page accueil département
    """
    return url_for("scolar.index_html", scodoc_dept=g.scodoc_dept)[
        : -len("/index_html")
    ]


def NotesURL():
    """URL of Notes
    e.g. https://scodoc.xxx.fr/ScoDoc/DEPT/Scolarite/Notes
    = url de base des méthodes de notes
    (page accueil programmes).
    """
    return url_for("notes.index_html", scodoc_dept=g.scodoc_dept)[: -len("/index_html")]


def AbsencesURL():
    """URL of Absences"""
    return url_for("absences.index_html", scodoc_dept=g.scodoc_dept)[
        : -len("/index_html")
    ]


def AssiduitesURL():
    """URL of Assiduités"""
    return url_for("assiduites.bilan_dept", scodoc_dept=g.scodoc_dept)[
        : -len("/BilanDept")
    ]


def UsersURL():
    """URL of Users
    e.g. https://scodoc.xxx.fr/ScoDoc/DEPT/Scolarite/Users
    = url de base des requêtes ZScoUsers
    et page accueil users
    """
    return url_for("users.index_html", scodoc_dept=g.scodoc_dept)[: -len("/index_html")]


# ---- Simple python utilities


def simplesqlquote(s, maxlen=50):
    """simple SQL quoting to avoid most SQL injection attacks.
    Note: we use this function in the (rare) cases where we have to
    construct SQL code manually"""
    s = s[:maxlen]
    s.replace("'", r"\'")
    s.replace(";", r"\;")
    for bad in ("select", "drop", ";", "--", "insert", "delete", "xp_"):
        s = s.replace(bad, "")
    return s


def unescape_html(s):
    """un-escape html entities"""
    s = s.strip().replace("&amp;", "&")
    s = s.replace("&lt;", "<")
    s = s.replace("&gt;", ">")
    return s


def build_url_query(url: str, **params) -> str:
    """Add parameters to existing url, as a query string"""
    url_parse = urlparse(url)
    query = url_parse.query
    url_dict = dict(parse_qsl(query))
    url_dict.update(params)
    url_new_query = urlencode(url_dict)
    url_parse = url_parse._replace(query=url_new_query)
    new_url = urlunparse(url_parse)
    return new_url


# test if obj is iterable (but not a string)
isiterable = lambda obj: getattr(obj, "__iter__", False)


def unescape_html_dict(d):
    """un-escape all dict values, recursively"""
    try:
        indices = list(d.keys())
    except:
        indices = list(range(len(d)))
    for k in indices:
        v = d[k]
        if isinstance(v, bytes):
            d[k] = unescape_html(v)
        elif isiterable(v):
            unescape_html_dict(v)


# Expressions used to check noms/prenoms
FORBIDDEN_CHARS_EXP = re.compile(r"[*\|~\(\)\\]")
ALPHANUM_EXP = re.compile(r"^[\w-]+$", re.UNICODE)


def is_valid_code_nip(s):
    """True si s peut être un code NIP: au moins 6 chiffres décimaux"""
    if not s:
        return False
    return re.match(r"^[0-9]{6,32}$", s)


def strnone(s):
    "convert s to string, '' if s is false"
    if s:
        return str(s)
    else:
        return ""


def strip_str(s):
    "if s is a string, strip it, if is None, do nothing"
    return s.strip() if s else s


def stripquotes(s):
    "strip s from spaces and quotes"
    s = s.strip()
    if s and ((s[0] == '"' and s[-1] == '"') or (s[0] == "'" and s[-1] == "'")):
        s = s[1:-1]
    return s


def suppress_accents(s):
    "remove accents and suppress non ascii characters from string s"
    if isinstance(s, str):
        return (
            unicodedata.normalize("NFD", s)
            .encode("ascii", "ignore")
            .decode(SCO_ENCODING)
        )
    return s  # may be int


class PurgeChars:
    """delete all chars except those belonging to the specified string"""

    def __init__(self, allowed_chars=""):
        self.allowed_chars_set = {ord(c) for c in allowed_chars}

    def __getitem__(self, x):
        if x not in self.allowed_chars_set:
            return None
        raise LookupError()


def purge_chars(s, allowed_chars=""):
    return s.translate(PurgeChars(allowed_chars=allowed_chars))


def sanitize_string(s, remove_spaces=True):
    """s is an ordinary string, encoding given by SCO_ENCODING"
    suppress accents and chars interpreted in XML
    Irreversible (not a quote)

    For ids and some filenames
    """
    # Table suppressing some chars:
    to_del = "'`\"<>!&\\ " if remove_spaces else "'`\"<>!&"
    trans = str.maketrans("", "", to_del)

    return suppress_accents(s.translate(trans)).replace("\t", "_")


_BAD_FILENAME_CHARS = str.maketrans("", "", ":/\\&[]*?'")


def make_filename(name):
    """Try to convert name to a reasonable filename
    without spaces, (back)slashes, : and without accents
    """
    return (
        suppress_accents(name.translate(_BAD_FILENAME_CHARS)).replace(" ", "_")
        or "scodoc"
    )


VALID_CARS = (
    "-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.!"  # no / !
)
VALID_CARS_SET = set(VALID_CARS)
VALID_EXP = re.compile("^[" + VALID_CARS + "]+$")


def sanitize_filename(filename):
    """Keep only valid chars
    used for archives filenames
    """
    filename = suppress_accents(filename.replace(" ", "_"))
    sane = "".join([c for c in filename if c in VALID_CARS_SET])
    if len(sane) < 2:
        sane = time.strftime("%Y-%m-%d-%H%M%S") + "-" + sane
    return sane


def is_valid_filename(filename):
    """True if filename is safe"""
    return VALID_EXP.match(filename)


BOOL_STR = {
    "": False,
    "0": False,
    "1": True,
    "f": False,
    "false": False,
    "n": False,
    "t": True,
    "true": True,
    "y": True,
}


def to_bool(x) -> bool:
    """Cast value to boolean.
    The value may be encoded as a string
    False are: empty, "0", "False", "f", "n".
    True: all other values, such as "1", "True", "foo", "bar"...
    Case insentive, ignore leading and trailing spaces.
    """
    if isinstance(x, str):
        return BOOL_STR.get(x.lower().strip(), True)
    return bool(x)


# Min/Max values for numbers stored in database:
DB_MIN_FLOAT = -1e30
DB_MAX_FLOAT = 1e30
DB_MIN_INT = -(1 << 31)
DB_MAX_INT = (1 << 31) - 1


def bul_filename_old(sem: dict, etud: dict, fmt):
    """Build a filename for this bulletin"""
    dt = time.strftime("%Y-%m-%d")
    filename = f"bul-{sem['titre_num']}-{dt}-{etud['nom']}.{fmt}"
    filename = make_filename(filename)
    return filename


def bul_filename(formsemestre, etud, prefix="bul"):
    """Build a filename for this bulletin (without suffix)"""
    dt = time.strftime("%Y-%m-%d")
    filename = f"{prefix}-{formsemestre.titre_num()}-{dt}-{etud.nom}"
    filename = make_filename(filename)
    return filename


def flash_errors(form):
    """Flashes form errors (version sommaire)"""
    for field, errors in form.errors.items():
        flash(
            "Erreur: voir le champs %s" % (getattr(form, field).label.text,),
            "warning",
        )
        # see https://getbootstrap.com/docs/4.0/components/alerts/


def flash_once(message: str):
    """Flash the message, but only once per request"""
    if not hasattr(g, "sco_flashed_once"):
        g.sco_flashed_once = set()
    if not message in g.sco_flashed_once:
        flash(message)
        g.sco_flashed_once.add(message)


def sendCSVFile(data, filename):  # DEPRECATED  utiliser send_file
    """publication fichier CSV."""
    return send_file(data, filename=filename, mime=CSV_MIMETYPE, attached=True)


def sendPDFFile(data, filename):  # DEPRECATED  utiliser send_file
    return send_file(data, filename=filename, mime=PDF_MIMETYPE, attached=True)


def sendJSON(data, attached=False, filename=None):
    js = json.dumps(data, indent=1, cls=ScoDocJSONEncoder)
    return send_file(
        js, filename=filename or "sco_data.json", mime=JSON_MIMETYPE, attached=attached
    )


def sendXML(
    data,
    tagname=None,
    force_outer_xml_tag=True,
    attached=False,
    quote=False,
    filename=None,
):
    if type(data) != list:
        data = [data]  # always list-of-dicts
    if force_outer_xml_tag:
        data = [{tagname: data}]
        tagname += "_list"
    doc = sco_xml.simple_dictlist2xml(data, tagname=tagname, quote=quote)
    return send_file(
        doc, filename=filename or "sco_data.xml", mime=XML_MIMETYPE, attached=attached
    )


def sendResult(
    data,
    name=None,
    fmt=None,
    force_outer_xml_tag=True,
    attached=False,
    quote_xml=False,
    filename=None,
):
    if (fmt is None) or (fmt == "html"):
        return data
    elif fmt == "xml":  # name is outer tagname
        return sendXML(
            data,
            tagname=name,
            force_outer_xml_tag=force_outer_xml_tag,
            attached=attached,
            quote=quote_xml,
            filename=filename,
        )
    elif fmt == "json":
        return sendJSON(data, attached=attached, filename=filename)
    else:
        raise ValueError(f"invalid format: {fmt}")


def send_file(data, filename="", suffix="", mime=None, attached=None):
    """Build Flask Response for file download of given type
    By default (attached is None), json and xml are inlined and other types are attached.
    """
    if attached is None:
        if mime == XML_MIMETYPE or mime == JSON_MIMETYPE:
            attached = False
        else:
            attached = True
    if filename:
        if suffix:
            filename += suffix
        filename = make_filename(filename)
    response = make_response(data)
    response.headers["Content-Type"] = mime
    if attached and filename:
        response.headers["Content-Disposition"] = 'attachment; filename="%s"' % filename
    return response


def send_docx(document, filename):
    "Send a python-docx document"
    buffer = io.BytesIO()  # in-memory document, no disk file
    document.save(buffer)
    buffer.seek(0)
    return flask.send_file(
        buffer,
        download_name=sanitize_filename(filename),
        mimetype=DOCX_MIMETYPE,
    )


def get_request_args():
    """returns a dict with request (POST or GET) arguments
    converted to suit legacy Zope style (scodoc7) functions.
    """
    # copy to get a mutable object (necessary for TrivialFormulator and several methods)
    if request.method == "POST":
        # request.form is a werkzeug.datastructures.ImmutableMultiDict
        # must copy to get a mutable version (needed by TrivialFormulator)
        vals = request.form.copy()
        if request.files:
            # Add files in form:
            vals.update(request.files)
        for k in request.form:
            if k.endswith(":list"):
                vals[k[:-5]] = request.form.getlist(k)
    elif request.method == "GET":
        vals = {}
        for k in request.args:
            # current_app.logger.debug("%s\t%s" % (k, request.args.getlist(k)))
            if k.endswith(":list"):
                vals[k[:-5]] = request.args.getlist(k)
            else:
                values = request.args.getlist(k)
                vals[k] = values[0] if len(values) == 1 else values
    return vals


def json_error(status_code, message=None) -> Response:
    """Simple JSON for errors."""
    payload = {
        "error": HTTP_STATUS_CODES.get(status_code, "Unknown error"),
        "status": status_code,
    }
    if message:
        payload["message"] = message
    response = json_response(status_=status_code, data_=payload)
    response.status_code = status_code
    log(f"Error: {response}")
    return response


def json_ok_response(status_code=200, payload=None) -> Response:
    """Simple JSON respons for "success" """
    payload = payload or {"OK": True}
    response = json_response(status_=status_code, data_=payload)
    response.status_code = status_code
    return response


def get_scodoc_version():
    "return a string identifying ScoDoc version"
    return sco_version.SCOVERSION


def check_scodoc7_password(scodoc7_hash, password):
    """Check a password vs scodoc7 hash
    used only during old databases migrations"""
    m = md5()
    m.update(password.encode("utf-8"))
    h = base64.encodebytes(m.digest()).decode("utf-8").strip()
    return h == scodoc7_hash


# Simple string manipulations


def abbrev_prenom(prenom):
    "Donne l'abreviation d'un prenom"
    # un peu lent, mais espère traiter tous les cas
    # Jean -> J.
    # Charles -> Ch.
    # Jean-Christophe -> J.-C.
    # Marie Odile -> M. O.
    prenom = prenom.replace(".", " ").strip()
    if not prenom:
        return ""
    d = prenom[:3].upper()
    if d == "CHA":
        abrv = "Ch."  # 'Charles' donne 'Ch.'
        i = 3
    else:
        abrv = prenom[0].upper() + "."
        i = 1
    n = len(prenom)
    while i < n:
        c = prenom[i]
        if c == " " or c == "-" and i < n - 1:
            sep = c
            i += 1
            # gobbe tous les separateurs
            while i < n and (prenom[i] == " " or prenom[i] == "-"):
                if prenom[i] == "-":
                    sep = "-"
                i += 1
            if i < n:
                abrv += sep + prenom[i].upper() + "."
        i += 1
    return abrv


#
def timedate_human_repr():
    "representation du temps courant pour utilisateur"
    return time.strftime("%d/%m/%Y à %Hh%M")


def annee_scolaire_repr(year, month):
    """representation de l'annee scolaire : '2009 - 2010'
    à partir d'une date.
    """
    if month >= MONTH_DEBUT_ANNEE_SCOLAIRE:  # apres le 1er aout
        return f"{year} - {year + 1}"
    else:
        return f"{year - 1} - {year}"


def annee_scolaire() -> int:
    """Année de debut de l'annee scolaire courante"""
    t = time.localtime()
    year, month = t[0], t[1]
    return annee_scolaire_debut(year, month)


def annee_scolaire_debut(year, month) -> int:
    """Annee scolaire de début.
    Par défaut (hémisphère nord), l'année du mois de août
    précédent la date indiquée.
    """
    if int(month) >= MONTH_DEBUT_ANNEE_SCOLAIRE:
        return int(year)
    else:
        return int(year) - 1


def date_debut_anne_scolaire(annee_sco: int) -> datetime:
    """La date de début de l'année scolaire
    (par défaut, le 1er aout)
    """
    return datetime.datetime(year=annee_sco, month=MONTH_DEBUT_ANNEE_SCOLAIRE, day=1)


def date_fin_anne_scolaire(annee_sco: int) -> datetime:
    """La date de fin de l'année scolaire
    (par défaut, le 31 juillet de l'année suivante)
    """
    # on prend la date de début de l'année scolaire suivante,
    # et on lui retre 1 jour.
    # On s'affranchit ainsi des problèmes de durées de mois.
    return datetime.datetime(
        year=annee_sco + 1, month=MONTH_DEBUT_ANNEE_SCOLAIRE, day=1
    ) - datetime.timedelta(days=1)


def sem_decale_str(sem):
    """'D' si semestre decalé, ou ''"""
    # considère "décalé" les semestre impairs commençant entre janvier et juin
    # et les pairs entre juillet et decembre
    if sem["semestre_id"] <= 0:
        return ""
    if (sem["semestre_id"] % 2 and sem["mois_debut_ord"] <= 6) or (
        not sem["semestre_id"] % 2 and sem["mois_debut_ord"] > 6
    ):
        return "D"
    else:
        return ""


def is_valid_mail(email):
    """True if well-formed email address"""
    return re.match(r"^.+@.+\..{2,3}$", email)


def graph_from_edges(edges, graph_name="mygraph"):
    """Crée un graph pydot
    à partir d'une liste d'arêtes [ (n1, n2), (n2, n3), ... ]
    où n1, n2, ... sont des chaînes donnant l'id des nœuds.

    Fonction remplaçant celle de pydot qui est buggée.
    """
    nodes = set([it for tup in edges for it in tup])
    graph = pydot.Dot(graph_name)
    for n in nodes:
        graph.add_node(pydot.Node(n))
    for e in edges:
        graph.add_edge(pydot.Edge(src=e[0], dst=e[1]))
    return graph


ICONSIZES = {}  # name : (width, height) cache image sizes


def icontag(name, file_format="png", no_size=False, **attrs):
    """tag HTML pour un icone.
    (dans les versions anterieures on utilisait Zope)
    Les icones sont des fichiers PNG dans .../static/icons
    Si la taille (width et height) n'est pas spécifiée, lit l'image
    pour la mesurer (et cache le résultat).
    """
    if (not no_size) and (("width" not in attrs) or ("height" not in attrs)):
        if name not in ICONSIZES:
            img_file = os.path.join(
                Config.SCODOC_DIR,
                "app/static/icons/%s.%s"
                % (
                    name,
                    file_format,
                ),
            )
            with PILImage.open(img_file) as image:
                width, height = image.size[0], image.size[1]
            ICONSIZES[name] = (width, height)  # cache
        else:
            width, height = ICONSIZES[name]
        attrs["width"] = width
        attrs["height"] = height
    if "border" not in attrs:
        attrs["border"] = 0
    if "alt" not in attrs:
        attrs["alt"] = "logo %s" % name
    s = " ".join(['%s="%s"' % (k, attrs[k]) for k in attrs])
    return f'<img class="{name}" {s} src="{STATIC_DIR}/icons/{name}.{file_format}" />'


ICON_PDF = icontag("pdficon16x20_img", title="Version PDF")
ICON_XLS = icontag("xlsicon_img", title="Version tableur")

# HTML emojis
EMO_WARNING = "&#9888;&#65039;"  # warning /!\
EMO_RED_TRIANGLE_DOWN = "&#128315;"  # red triangle pointed down
EMO_PREV_ARROW = "&#10094;"
EMO_NEXT_ARROW = "&#10095;"


def sort_dates(L, reverse=False):
    """Return sorted list of dates, allowing None items (they are put at the beginning)"""
    mindate = datetime.datetime(datetime.MINYEAR, 1, 1)
    try:
        return sorted(L, key=lambda x: x or mindate, reverse=reverse)
    except:
        # Helps debugging
        log("sort_dates( %s )" % L)
        raise


def heterogeneous_sorting_key(x):
    "key to sort non homogeneous sequences"
    return (float(x), "") if isinstance(x, (bool, float, int)) else (-1e34, str(x))


def query_portal(req, msg="Portail Apogee", timeout=3):
    """Retreives external data using HTTP request
    (used to connect to Apogee portal, or ScoDoc server)
    returns a string,  "" on error
    """
    log("query_portal: %s" % req)
    error_message = None
    try:
        r = requests.get(req, timeout=timeout)  # seconds / request
    except requests.ConnectionError:
        error_message = "ConnectionError"
    except requests.Timeout:
        error_message = "Timeout"
    except requests.TooManyRedirects:
        error_message = "TooManyRedirects"
    except requests.RequestException:
        error_message = f"can't connect to {msg}"
    if error_message is not None:
        log(f"query_portal: {error_message}")
        return ""
    if r.status_code != 200:
        log(f"query_portal: http error {r.status_code}")
        return ""

    return r.text


def confirm_dialog(
    message="<p>Confirmer ?</p>",
    OK="OK",
    add_headers=True,  # complete page
    cancel_label="Annuler",
    cancel_url="",
    dest_url="",
    help_msg=None,
    parameters: dict = None,
    target_variable="dialog_confirmed",
):
    """HTML confirmation dialog: submit (POST) to same page or dest_url if given."""
    from app.scodoc import html_sco_header

    parameters = parameters or {}
    # dialog de confirmation simple
    parameters[target_variable] = 1
    # Attention: la page a pu etre servie en GET avec des parametres
    # si on laisse l'url "action" vide, les parametres restent alors que l'on passe en POST...
    if not dest_url:
        action = ""
    else:
        # strip remaining parameters from destination url:
        dest_url = urllib.parse.splitquery(dest_url)[0]
        action = f'action="{dest_url}"'

    H = [
        f"""<form {action} method="POST">
        {message}
        """,
    ]
    if OK or not cancel_url:
        H.append(f'<input type="submit" value="{OK}"/>')
    if cancel_url:
        H.append(
            f"""<input type ="button" value="{cancel_label}"
            onClick="document.location='{cancel_url}';"/>"""
        )
    for param in parameters.keys():
        if parameters[param] is None:
            parameters[param] = ""
        if isinstance(parameters[param], list):
            for e in parameters[param]:
                H.append(f"""<input type="hidden" name="{param}" value="{e}"/>""")
        else:
            H.append(
                f"""<input type="hidden" name="{param}" value="{parameters[param]}"/>"""
            )
    H.append("</form>")
    if help_msg:
        H.append('<p class="help">' + help_msg + "</p>")
    if add_headers:
        return (
            html_sco_header.sco_header() + "\n".join(H) + html_sco_header.sco_footer()
        )
    else:
        return "\n".join(H)


def objects_renumber(db, obj_list) -> None:
    """fixe les numeros des objets d'une liste de modèles
    pour ne pas changer son ordre"""
    log("objects_renumber")
    for i, obj in enumerate(obj_list):
        obj.numero = i
        db.session.add(obj)
    db.session.commit()


def comp_ranks(tab: list[tuple]) -> dict[int, str]:
    """Calcul rangs à partir d'une liste ordonnée de tuples [ (valeur, ..., etudid) ]
    (valeur est une note numérique), en tenant compte des ex-aequos
    Le resultat est: { etudid : rang } où rang est une chaine decrivant le rang
    """
    rangs = {}  # { etudid : rang } (rang est une chaine)
    nb_ex = 0  # nb d'ex-aequo consécutifs en cours
    for i, row in enumerate(tab):
        # test ex-aequo
        if i < len(tab) - 1:
            next_val = tab[i + 1][0]
        else:
            next_val = None
        moy = row[0]
        if nb_ex:
            srang = "%d ex" % (i + 1 - nb_ex)
            if moy == next_val:
                nb_ex += 1
            else:
                nb_ex = 0
        else:
            if moy == next_val:
                srang = "%d ex" % (i + 1 - nb_ex)
                nb_ex = 1
            else:
                srang = "%d" % (i + 1)
        rangs[row[-1]] = srang
    return rangs


def gen_cell(key: str, row: dict, elt="td", with_col_class=False):
    "html table cell"
    klass = row.get(f"_{key}_class", "")
    if with_col_class:
        klass = key + " " + klass
    attrs = f'class="{klass}"' if klass else ""
    if elt == "th":
        attrs += ' scope="row"'
    data = row.get(f"_{key}_data")  # dict
    if data:
        for k in data:
            attrs += f' data-{k}="{data[k]}"'
    order = row.get(f"_{key}_order")
    if order:
        attrs += f' data-order="{order}"'
    content = row.get(key, "")
    target = row.get(f"_{key}_target")
    target_attrs = row.get(f"_{key}_target_attrs", "")
    if target or target_attrs:  # avec lien
        href = f'href="{target}"' if target else ""
        content = f"<a {href} {target_attrs}>{content}</a>"
    return f"<{elt} {attrs}>{content}</{elt}>"


def gen_row(
    keys: list[str], row, elt="td", selected_etudid=None, with_col_classes=False
):
    "html table row"
    klass = row.get("_tr_class")
    if row.get("etudid", "") == selected_etudid:
        klass += " row_selected"
    tr_class = f'class="{klass}"' if klass else ""
    return f"""<tr {tr_class}>{
        "".join([gen_cell(key, row, elt, with_col_class=with_col_classes)
        for key in keys if not key.startswith('_')])
        }</tr>"""


# Pour accès depuis les templates jinja
def is_entreprises_enabled():
    from app.models import ScoDocSiteConfig

    return ScoDocSiteConfig.is_entreprises_enabled()