ScoDoc/app/scodoc/sco_utils.py

# -*- mode: python -*-
# -*- coding: utf-8 -*-

##############################################################################
#
# Gestion scolarite IUT
#
# Copyright (c) 1999 - 2023 Emmanuel Viennet.  All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#   Emmanuel Viennet      emmanuel.viennet@viennet.net
#
##############################################################################


""" Common definitions
"""
import base64
import bisect
import collections
import datetime
from enum import IntEnum
import io
import json
from hashlib import md5
import numbers
import os
import re
import _thread
import time
import unicodedata
import urllib
from urllib.parse import urlparse, parse_qsl, urlunparse, urlencode

import numpy as np
from PIL import Image as PILImage
import pydot
import requests

import flask
from flask import g, request
from flask import flash, url_for, make_response, jsonify
from werkzeug.http import HTTP_STATUS_CODES

from config import Config
from app import log
from app.scodoc.sco_vdi import ApoEtapeVDI
from app.scodoc.codes_cursus import NOTES_TOLERANCE, CODES_EXPL
from app.scodoc import sco_xml
import sco_version

# le répertoire static, lié à chaque release pour éviter les problèmes de caches
STATIC_DIR = "/ScoDoc/static/links/" + sco_version.SCOVERSION

# ----- CALCUL ET PRESENTATION DES NOTES
NOTES_PRECISION = 1e-4  # evite eventuelles erreurs d'arrondis
NOTES_MIN = 0.0  # valeur minimale admise pour une note (sauf malus, dans [-20, 20])
NOTES_MAX = 1000.0
NOTES_ABSENCE = -999.0  # absences dans les DataFrames, NULL en base
NOTES_NEUTRALISE = -1000.0  # notes non prises en comptes dans moyennes
NOTES_SUPPRESS = -1001.0  # note a supprimer
NOTES_ATTENTE = -1002.0  # note "en attente" (se calcule comme une note neutralisee)

NO_NOTE_STR = "-"  # contenu des cellules de tableaux html sans notes

# ---- CODES INSCRIPTION AUX SEMESTRES
# (champ etat de FormSemestreInscription)
INSCRIT = "I"
DEMISSION = "D"
DEF = "DEF"
ETATS_INSCRIPTION = {
    INSCRIT: "Inscrit",
    DEMISSION: "Démission",
    DEF: "Défaillant",
}


# Types de modules
class ModuleType(IntEnum):
    """Code des types de module."""

    # Stockés en BD dans Module.module_type: ne pas modifier ces valeurs
    STANDARD = 0
    MALUS = 1
    RESSOURCE = 2  # BUT
    SAE = 3  # BUT

    @classmethod
    def get_abbrev(cls, code) -> str:
        """Abbréviation décrivant le type de module à partir du code integer:
        "mod", "malus", "res", "sae"
        (utilisées pour style CSS)
        """
        return {
            ModuleType.STANDARD: "mod",
            ModuleType.MALUS: "malus",
            ModuleType.RESSOURCE: "res",
            ModuleType.SAE: "sae",
        }.get(code, "???")


MODULE_TYPE_NAMES = {
    ModuleType.STANDARD: "Module",
    ModuleType.MALUS: "Malus",
    ModuleType.RESSOURCE: "Ressource",
    ModuleType.SAE: "SAÉ",
    None: "Module",
}

PARTITION_PARCOURS = "Parcours"

MALUS_MAX = 20.0
MALUS_MIN = -20.0

APO_MISSING_CODE_STR = "----"  # shown in HTML pages in place of missing code Apogée
EDIT_NB_ETAPES = 6  # Nombre max de codes étapes / semestre presentés dans l'UI

IT_SITUATION_MISSING_STR = (
    "____"  # shown on ficheEtud (devenir) in place of empty situation
)

RANG_ATTENTE_STR = "(attente)"  #  rang affiché sur bulletins quand notes en attente

# borne supérieure de chaque mention
NOTES_MENTIONS_TH = (
    NOTES_TOLERANCE,
    7.0,
    10.0,
    12.0,
    14.0,
    16.0,
    18.0,
    20.0 + NOTES_TOLERANCE,
)
NOTES_MENTIONS_LABS = (
    "Nul",
    "Faible",
    "Insuffisant",
    "Passable",
    "Assez bien",
    "Bien",
    "Très bien",
    "Excellent",
)

EVALUATION_NORMALE = 0
EVALUATION_RATTRAPAGE = 1
EVALUATION_SESSION2 = 2

# Dates et années scolaires
# Ces dates "pivot" sont paramétrables dans les préférences générales
# on donne ici les valeurs par défaut.
# Les semestres commençant à partir du 1er août 20XX sont
# dans l'année scolaire 20XX
MONTH_DEBUT_ANNEE_SCOLAIRE = 8  # août
# Les semestres commençant à partir du 1er décembre
# sont "2eme période" (S_pair):
MONTH_DEBUT_PERIODE2 = MONTH_DEBUT_ANNEE_SCOLAIRE + 4

MONTH_NAMES_ABBREV = (
    "Jan ",
    "Fév ",
    "Mars",
    "Avr ",
    "Mai ",
    "Juin",
    "Jul ",
    "Août",
    "Sept",
    "Oct ",
    "Nov ",
    "Déc ",
)

MONTH_NAMES = (
    "janvier",
    "février",
    "mars",
    "avril",
    "mai",
    "juin",
    "juillet",
    "août",
    "septembre",
    "octobre",
    "novembre",
    "décembre",
)
DAY_NAMES = ("lundi", "mardi", "mercredi", "jeudi", "vendredi", "samedi", "dimanche")


def fmt_note(val, note_max=None, keep_numeric=False):
    """conversion note en str pour affichage dans tables HTML ou PDF.
    Si keep_numeric, laisse les valeur numeriques telles quelles (pour export Excel)
    """
    if val is None or val == NOTES_ABSENCE:
        return "ABS"
    if val == NOTES_NEUTRALISE:
        return "EXC"  # excuse, note neutralise
    if val == NOTES_ATTENTE:
        return "ATT"  # attente, note neutralisee
    if not isinstance(val, str):
        if np.isnan(val):
            return "~"
        if (note_max is not None) and note_max > 0:
            val = val * 20.0 / note_max
        if keep_numeric:
            return val
        else:
            s = "%2.2f" % round(float(val), 2)  # 2 chiffres apres la virgule
            s = "0" * (5 - len(s)) + s  # padding: 0 à gauche pour longueur 5: "12.34"
            return s
    else:
        return val.replace("NA", "-")


def fmt_coef(val):
    """Conversion valeur coefficient (float) en chaine"""
    if val < 0.01:
        return "%g" % val  # unusually small value
    return "%g" % round(val, 2)


def fmt_abs(val):
    """Conversion absences en chaine. val est une list [nb_abs_total, nb_abs_justifiees
    => NbAbs / Nb_justifiees
    """
    return "%s / %s" % (val[0], val[1])


def isnumber(x):
    "True if x is a number (int, float, etc.)"
    return isinstance(x, numbers.Number)


def jsnan(x):
    "if x is NaN, returns None"
    if isinstance(x, numbers.Number) and np.isnan(x):
        return None
    return x


def join_words(*words):
    words = [str(w).strip() for w in words if w is not None]
    return " ".join([w for w in words if w])


def get_mention(moy):
    """Texte "mention" en fonction de la moyenne générale"""
    try:
        moy = float(moy)
    except:
        return ""
    if moy > 0.0:
        return NOTES_MENTIONS_LABS[bisect.bisect_right(NOTES_MENTIONS_TH, moy)]
    else:
        return ""


def group_by_key(d: dict, key) -> dict:
    grouped = collections.defaultdict(lambda: [])
    for e in d:
        grouped[e[key]].append(e)
    return grouped


# ----- Global lock for critical sections (except notes_tables caches)
GSL = _thread.allocate_lock()  # Global ScoDoc Lock

SCODOC_DIR = Config.SCODOC_DIR

# ----- Repertoire "config" modifiable
#        /opt/scodoc-data/config
SCODOC_CFG_DIR = os.path.join(Config.SCODOC_VAR_DIR, "config")
# ----- Version information
SCODOC_VERSION_DIR = os.path.join(SCODOC_CFG_DIR, "version")
# ----- Repertoire tmp : /opt/scodoc-data/tmp
SCO_TMP_DIR = os.path.join(Config.SCODOC_VAR_DIR, "tmp")
if not os.path.exists(SCO_TMP_DIR) and os.path.exists(Config.SCODOC_VAR_DIR):
    os.mkdir(SCO_TMP_DIR, 0o755)
# ----- Les logos: /opt/scodoc-data/config/logos
SCODOC_LOGOS_DIR = os.path.join(SCODOC_CFG_DIR, "logos")
LOGOS_IMAGES_ALLOWED_TYPES = ("jpg", "jpeg", "png")  # remind that PIL does not read pdf
LOGOS_DIR_PREFIX = "logos_"
LOGO_FILE_PREFIX = "logo_"

# forme générale des noms des fichiers logos/background:
# SCODOC_LOGO_DIR/LOGO_FILE_PREFIX<name>.<suffix>   (fichier global) ou
# SCODOC_LOGO_DIR/LOGOS_DIR_PREFIX<dept_id>/LOGO_FILE_PREFIX<name>.<suffix> (fichier départemental)

# ----- Les outils distribués
SCO_TOOLS_DIR = os.path.join(Config.SCODOC_DIR, "tools")


# ----- Lecture du fichier de configuration
from app.scodoc import sco_config
from app.scodoc import sco_config_load

sco_config_load.load_local_configuration(SCODOC_CFG_DIR)
CONFIG = sco_config.CONFIG
if hasattr(CONFIG, "CODES_EXPL"):
    CODES_EXPL.update(
        CONFIG.CODES_EXPL
    )  # permet de customiser les explications de codes

if CONFIG.CUSTOM_HTML_HEADER:
    CUSTOM_HTML_HEADER = open(CONFIG.CUSTOM_HTML_HEADER).read()
else:
    CUSTOM_HTML_HEADER = ""

if CONFIG.CUSTOM_HTML_HEADER_CNX:
    CUSTOM_HTML_HEADER_CNX = open(CONFIG.CUSTOM_HTML_HEADER_CNX).read()
else:
    CUSTOM_HTML_HEADER_CNX = ""

if CONFIG.CUSTOM_HTML_FOOTER:
    CUSTOM_HTML_FOOTER = open(CONFIG.CUSTOM_HTML_FOOTER).read()
else:
    CUSTOM_HTML_FOOTER = ""

if CONFIG.CUSTOM_HTML_FOOTER_CNX:
    CUSTOM_HTML_FOOTER_CNX = open(CONFIG.CUSTOM_HTML_FOOTER_CNX).read()
else:
    CUSTOM_HTML_FOOTER_CNX = ""

SCO_ENCODING = "utf-8"  # used by Excel, XML, PDF, ...


SCO_DEFAULT_SQL_USER = "scodoc"  # should match Zope process UID
SCO_DEFAULT_SQL_PORT = "5432"
SCO_DEFAULT_SQL_USERS_CNX = "dbname=SCOUSERS port=%s" % SCO_DEFAULT_SQL_PORT

# Valeurs utilisées pour affichage seulement, pas de requetes ni de mails envoyés:
SCO_WEBSITE = "https://scodoc.org"
SCO_USER_MANUAL = "https://scodoc.org/GuideUtilisateur"
SCO_ANNONCES_WEBSITE = "https://listes.univ-paris13.fr/mailman/listinfo/scodoc-annonces"
SCO_DEVEL_LIST = "scodoc-devel@listes.univ-paris13.fr"
SCO_USERS_LIST = "notes@listes.univ-paris13.fr"
SCO_LISTS_URL = "https://scodoc.org/Contact"
SCO_DISCORD_ASSISTANCE = "https://discord.gg/ybw6ugtFsZ"

# Mails avec exceptions (erreurs) anormales envoyés à cette adresse:
# mettre '' pour désactiver completement l'envois de mails d'erreurs.
# (ces mails sont précieux pour corriger les erreurs, ne les désactiver que si
#  vous avez de bonnes raisons de le faire: vous pouvez me contacter avant)
SCO_EXC_MAIL = "scodoc-exception@viennet.net"

# L'adresse du mainteneur (non utilisée automatiquement par ScoDoc: ne pas changer)
SCO_DEV_MAIL = "emmanuel.viennet@gmail.com"  # SVP ne pas changer

# Adresse pour l'envoi des dumps (pour assistance technnique):
#   ne pas changer (ou vous perdez le support)
SCO_DUMP_UP_URL = "https://scodoc.org/scodoc-installmgr/upload-dump"
SCO_UP2DATE = "https://scodoc.org/scodoc-installmgr/check_version"
CSV_FIELDSEP = ";"
CSV_LINESEP = "\n"
CSV_MIMETYPE = "text/comma-separated-values"
CSV_SUFFIX = ".csv"
DOCX_MIMETYPE = (
    "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
)
DOCX_SUFFIX = ".docx"
JSON_MIMETYPE = "application/json"
JSON_SUFFIX = ".json"
PDF_MIMETYPE = "application/pdf"
PDF_SUFFIX = ".pdf"
XLS_MIMETYPE = "application/vnd.ms-excel"
XLS_SUFFIX = ".xls"
XLSX_MIMETYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
XLSX_SUFFIX = ".xlsx"
XML_MIMETYPE = "text/xml"
XML_SUFFIX = ".xml"

# Format pour lesquels on exporte sans formattage des nombres (pas de perte de précision)
FORMATS_NUMERIQUES = {"csv", "xls", "xlsx", "xml", "json"}


def get_mime_suffix(format_code: str) -> tuple[str, str]:
    """Returns (MIME, SUFFIX) from format_code == "xls", "xml", ...
    SUFFIX includes the dot: ".xlsx", ".xml", ...
    "xls" and "xlsx" format codes give XLSX
    """
    d = {
        "csv": (CSV_MIMETYPE, CSV_SUFFIX),
        "docx": (DOCX_MIMETYPE, DOCX_SUFFIX),
        "xls": (XLSX_MIMETYPE, XLSX_SUFFIX),
        "xlsx": (XLSX_MIMETYPE, XLSX_SUFFIX),
        "pdf": (PDF_MIMETYPE, PDF_SUFFIX),
        "xml": (XML_MIMETYPE, XML_SUFFIX),
        "json": (JSON_MIMETYPE, JSON_SUFFIX),
    }
    return d[format_code]


# Admissions des étudiants
# Différents types de voies d'admission:
# (stocké en texte libre dans la base, mais saisie par menus pour harmoniser)
TYPE_ADMISSION_DEFAULT = "Inconnue"
TYPES_ADMISSION = (TYPE_ADMISSION_DEFAULT, "APB", "APB-PC", "CEF", "Direct")

BULLETINS_VERSIONS = ("short", "selectedevals", "long")

# Support for ScoDoc7 compatibility


def ScoURL():
    """base URL for this sco instance.
    e.g. https://scodoc.xxx.fr/ScoDoc/DEPT/Scolarite
    = page accueil département
    """
    return url_for("scolar.index_html", scodoc_dept=g.scodoc_dept)[
        : -len("/index_html")
    ]


def NotesURL():
    """URL of Notes
    e.g. https://scodoc.xxx.fr/ScoDoc/DEPT/Scolarite/Notes
    = url de base des méthodes de notes
    (page accueil programmes).
    """
    return url_for("notes.index_html", scodoc_dept=g.scodoc_dept)[: -len("/index_html")]


def AbsencesURL():
    """URL of Absences"""
    return url_for("absences.index_html", scodoc_dept=g.scodoc_dept)[
        : -len("/index_html")
    ]


def UsersURL():
    """URL of Users
    e.g. https://scodoc.xxx.fr/ScoDoc/DEPT/Scolarite/Users
    = url de base des requêtes ZScoUsers
    et page accueil users
    """
    return url_for("users.index_html", scodoc_dept=g.scodoc_dept)[: -len("/index_html")]


# ---- Simple python utilities


def simplesqlquote(s, maxlen=50):
    """simple SQL quoting to avoid most SQL injection attacks.
    Note: we use this function in the (rare) cases where we have to
    construct SQL code manually"""
    s = s[:maxlen]
    s.replace("'", r"\'")
    s.replace(";", r"\;")
    for bad in ("select", "drop", ";", "--", "insert", "delete", "xp_"):
        s = s.replace(bad, "")
    return s


def unescape_html(s):
    """un-escape html entities"""
    s = s.strip().replace("&amp;", "&")
    s = s.replace("&lt;", "<")
    s = s.replace("&gt;", ">")
    return s


def build_url_query(url: str, **params) -> str:
    """Add parameters to existing url, as a query string"""
    url_parse = urlparse(url)
    query = url_parse.query
    url_dict = dict(parse_qsl(query))
    url_dict.update(params)
    url_new_query = urlencode(url_dict)
    url_parse = url_parse._replace(query=url_new_query)
    new_url = urlunparse(url_parse)
    return new_url


# test if obj is iterable (but not a string)
isiterable = lambda obj: getattr(obj, "__iter__", False)


def unescape_html_dict(d):
    """un-escape all dict values, recursively"""
    try:
        indices = list(d.keys())
    except:
        indices = list(range(len(d)))
    for k in indices:
        v = d[k]
        if isinstance(v, bytes):
            d[k] = unescape_html(v)
        elif isiterable(v):
            unescape_html_dict(v)


# Expressions used to check noms/prenoms
FORBIDDEN_CHARS_EXP = re.compile(r"[*\|~\(\)\\]")
ALPHANUM_EXP = re.compile(r"^[\w-]+$", re.UNICODE)


def is_valid_code_nip(s):
    """True si s peut être un code NIP: au moins 6 chiffres décimaux"""
    if not s:
        return False
    return re.match(r"^[0-9]{6,32}$", s)


def strnone(s):
    "convert s to string, '' if s is false"
    if s:
        return str(s)
    else:
        return ""


def strip_str(s):
    "if s is a string, strip it, if is None, do nothing"
    return s.strip() if s else s


def stripquotes(s):
    "strip s from spaces and quotes"
    s = s.strip()
    if s and ((s[0] == '"' and s[-1] == '"') or (s[0] == "'" and s[-1] == "'")):
        s = s[1:-1]
    return s


def suppress_accents(s):
    "remove accents and suppress non ascii characters from string s"
    if isinstance(s, str):
        return (
            unicodedata.normalize("NFD", s)
            .encode("ascii", "ignore")
            .decode(SCO_ENCODING)
        )
    return s  # may be int


class PurgeChars:
    """delete all chars except those belonging to the specified string"""

    def __init__(self, allowed_chars=""):
        self.allowed_chars_set = {ord(c) for c in allowed_chars}

    def __getitem__(self, x):
        if x not in self.allowed_chars_set:
            return None
        raise LookupError()


def purge_chars(s, allowed_chars=""):
    return s.translate(PurgeChars(allowed_chars=allowed_chars))


def sanitize_string(s, remove_spaces=True):
    """s is an ordinary string, encoding given by SCO_ENCODING"
    suppress accents and chars interpreted in XML
    Irreversible (not a quote)

    For ids and some filenames
    """
    # Table suppressing some chars:
    to_del = "'`\"<>!&\\ " if remove_spaces else "'`\"<>!&"
    trans = str.maketrans("", "", to_del)

    return suppress_accents(s.translate(trans)).replace("\t", "_")


_BAD_FILENAME_CHARS = str.maketrans("", "", ":/\\&[]*?'")


def make_filename(name):
    """Try to convert name to a reasonable filename
    without spaces, (back)slashes, : and without accents
    """
    return (
        suppress_accents(name.translate(_BAD_FILENAME_CHARS)).replace(" ", "_")
        or "scodoc"
    )


VALID_CARS = (
    "-abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.!"  # no / !
)
VALID_CARS_SET = set(VALID_CARS)
VALID_EXP = re.compile("^[" + VALID_CARS + "]+$")


def sanitize_filename(filename):
    """Keep only valid chars
    used for archives filenames
    """
    filename = suppress_accents(filename.replace(" ", "_"))
    sane = "".join([c for c in filename if c in VALID_CARS_SET])
    if len(sane) < 2:
        sane = time.strftime("%Y-%m-%d-%H%M%S") + "-" + sane
    return sane


def is_valid_filename(filename):
    """True if filename is safe"""
    return VALID_EXP.match(filename)


BOOL_STR = {
    "": False,
    "0": False,
    "1": True,
    "f": False,
    "false": False,
    "n": False,
    "t": True,
    "true": True,
    "y": True,
}


def to_bool(x) -> bool:
    """Cast value to boolean.
    The value may be encoded as a string
    False are: empty, "0", "False", "f", "n".
    True: all other values, such as "1", "True", "foo", "bar"...
    Case insentive, ignore leading and trailing spaces.
    """
    if isinstance(x, str):
        return BOOL_STR.get(x.lower().strip(), True)
    return bool(x)


# Min/Max values for numbers stored in database:
DB_MIN_FLOAT = -1e30
DB_MAX_FLOAT = 1e30
DB_MIN_INT = -(1 << 31)
DB_MAX_INT = (1 << 31) - 1


def bul_filename_old(sem: dict, etud: dict, format):
    """Build a filename for this bulletin"""
    dt = time.strftime("%Y-%m-%d")
    filename = f"bul-{sem['titre_num']}-{dt}-{etud['nom']}.{format}"
    filename = make_filename(filename)
    return filename


def bul_filename(formsemestre, etud, format):
    """Build a filename for this bulletin"""
    dt = time.strftime("%Y-%m-%d")
    filename = f"bul-{formsemestre.titre_num()}-{dt}-{etud.nom}.{format}"
    filename = make_filename(filename)
    return filename


def flash_errors(form):
    """Flashes form errors (version sommaire)"""
    for field, errors in form.errors.items():
        flash(
            "Erreur: voir le champs %s" % (getattr(form, field).label.text,),
            "warning",
        )
        # see https://getbootstrap.com/docs/4.0/components/alerts/


def flash_once(message: str):
    """Flash the message, but only once per request"""
    if not hasattr(g, "sco_flashed_once"):
        g.sco_flashed_once = set()
    if not message in g.sco_flashed_once:
        flash(message)
        g.sco_flashed_once.add(message)


def sendCSVFile(data, filename):  # DEPRECATED  utiliser send_file
    """publication fichier CSV."""
    return send_file(data, filename=filename, mime=CSV_MIMETYPE, attached=True)


def sendPDFFile(data, filename):  # DEPRECATED  utiliser send_file
    return send_file(data, filename=filename, mime=PDF_MIMETYPE, attached=True)


class ScoDocJSONEncoder(json.JSONEncoder):
    def default(self, o):  # pylint: disable=E0202
        if isinstance(o, (datetime.date, datetime.datetime)):
            return o.isoformat()
        elif isinstance(o, ApoEtapeVDI):
            return str(o)
        else:
            return json.JSONEncoder.default(self, o)


def sendJSON(data, attached=False, filename=None):
    js = json.dumps(data, indent=1, cls=ScoDocJSONEncoder)
    return send_file(
        js, filename=filename or "sco_data.json", mime=JSON_MIMETYPE, attached=attached
    )


def sendXML(
    data,
    tagname=None,
    force_outer_xml_tag=True,
    attached=False,
    quote=False,
    filename=None,
):
    if type(data) != list:
        data = [data]  # always list-of-dicts
    if force_outer_xml_tag:
        data = [{tagname: data}]
        tagname += "_list"
    doc = sco_xml.simple_dictlist2xml(data, tagname=tagname, quote=quote)
    return send_file(
        doc, filename=filename or "sco_data.xml", mime=XML_MIMETYPE, attached=attached
    )


def sendResult(
    data,
    name=None,
    format=None,
    force_outer_xml_tag=True,
    attached=False,
    quote_xml=False,
    filename=None,
):
    if (format is None) or (format == "html"):
        return data
    elif format == "xml":  # name is outer tagname
        return sendXML(
            data,
            tagname=name,
            force_outer_xml_tag=force_outer_xml_tag,
            attached=attached,
            quote=quote_xml,
            filename=filename,
        )
    elif format == "json":
        return sendJSON(data, attached=attached, filename=filename)
    else:
        raise ValueError("invalid format: %s" % format)


def send_file(data, filename="", suffix="", mime=None, attached=None):
    """Build Flask Response for file download of given type
    By default (attached is None), json and xml are inlined and other types are attached.
    """
    if attached is None:
        if mime == XML_MIMETYPE or mime == JSON_MIMETYPE:
            attached = False
        else:
            attached = True
    if filename:
        if suffix:
            filename += suffix
        filename = make_filename(filename)
    response = make_response(data)
    response.headers["Content-Type"] = mime
    if attached and filename:
        response.headers["Content-Disposition"] = 'attachment; filename="%s"' % filename
    return response


def send_docx(document, filename):
    "Send a python-docx document"
    buffer = io.BytesIO()  # in-memory document, no disk file
    document.save(buffer)
    buffer.seek(0)
    return flask.send_file(
        buffer,
        download_name=sanitize_filename(filename),
        mimetype=DOCX_MIMETYPE,
    )


def get_request_args():
    """returns a dict with request (POST or GET) arguments
    converted to suit legacy Zope style (scodoc7) functions.
    """
    # copy to get a mutable object (necessary for TrivialFormulator and several methods)
    if request.method == "POST":
        # request.form is a werkzeug.datastructures.ImmutableMultiDict
        # must copy to get a mutable version (needed by TrivialFormulator)
        vals = request.form.copy()
        if request.files:
            # Add files in form:
            vals.update(request.files)
        for k in request.form:
            if k.endswith(":list"):
                vals[k[:-5]] = request.form.getlist(k)
    elif request.method == "GET":
        vals = {}
        for k in request.args:
            # current_app.logger.debug("%s\t%s" % (k, request.args.getlist(k)))
            if k.endswith(":list"):
                vals[k[:-5]] = request.args.getlist(k)
            else:
                values = request.args.getlist(k)
                vals[k] = values[0] if len(values) == 1 else values
    return vals


def json_error(status_code, message=None):
    """Simple JSON response, for errors"""
    payload = {
        "error": HTTP_STATUS_CODES.get(status_code, "Unknown error"),
        "status": status_code,
    }
    if message:
        payload["message"] = message
    response = jsonify(payload)
    response.status_code = status_code
    log(f"Error: {response}")
    return response


def json_ok_response(status_code=200, payload=None):
    """Simple JSON respons for "success" """
    payload = payload or {"OK": True}
    response = jsonify(payload)
    response.status_code = status_code
    return response


def get_scodoc_version():
    "return a string identifying ScoDoc version"
    return sco_version.SCOVERSION


def check_scodoc7_password(scodoc7_hash, password):
    """Check a password vs scodoc7 hash
    used only during old databases migrations"""
    m = md5()
    m.update(password.encode("utf-8"))
    h = base64.encodebytes(m.digest()).decode("utf-8").strip()
    return h == scodoc7_hash


# Simple string manipulations


def abbrev_prenom(prenom):
    "Donne l'abreviation d'un prenom"
    # un peu lent, mais espère traiter tous les cas
    # Jean -> J.
    # Charles -> Ch.
    # Jean-Christophe -> J.-C.
    # Marie Odile -> M. O.
    prenom = prenom.replace(".", " ").strip()
    if not prenom:
        return ""
    d = prenom[:3].upper()
    if d == "CHA":
        abrv = "Ch."  # 'Charles' donne 'Ch.'
        i = 3
    else:
        abrv = prenom[0].upper() + "."
        i = 1
    n = len(prenom)
    while i < n:
        c = prenom[i]
        if c == " " or c == "-" and i < n - 1:
            sep = c
            i += 1
            # gobbe tous les separateurs
            while i < n and (prenom[i] == " " or prenom[i] == "-"):
                if prenom[i] == "-":
                    sep = "-"
                i += 1
            if i < n:
                abrv += sep + prenom[i].upper() + "."
        i += 1
    return abrv


#
def timedate_human_repr():
    "representation du temps courant pour utilisateur"
    return time.strftime("%d/%m/%Y à %Hh%M")


def annee_scolaire_repr(year, month):
    """representation de l'annee scolaire : '2009 - 2010'
    à partir d'une date.
    """
    if month >= MONTH_DEBUT_ANNEE_SCOLAIRE:  # apres le 1er aout
        return f"{year} - {year + 1}"
    else:
        return f"{year - 1} - {year}"


def annee_scolaire() -> int:
    """Année de debut de l'annee scolaire courante"""
    t = time.localtime()
    year, month = t[0], t[1]
    return annee_scolaire_debut(year, month)


def annee_scolaire_debut(year, month) -> int:
    """Annee scolaire de début.
    Par défaut (hémisphère nord), l'année du mois de août
    précédent la date indiquée.
    """
    if int(month) >= MONTH_DEBUT_ANNEE_SCOLAIRE:
        return int(year)
    else:
        return int(year) - 1


def date_debut_anne_scolaire(annee_sco: int) -> datetime:
    """La date de début de l'année scolaire
    (par défaut, le 1er aout)
    """
    return datetime.datetime(year=annee_sco, month=MONTH_DEBUT_ANNEE_SCOLAIRE, day=1)


def date_fin_anne_scolaire(annee_sco: int) -> datetime:
    """La date de fin de l'année scolaire
    (par défaut, le 31 juillet de l'année suivante)
    """
    # on prend la date de début de l'année scolaire suivante,
    # et on lui retre 1 jour.
    # On s'affranchit ainsi des problèmes de durées de mois.
    return datetime.datetime(
        year=annee_sco + 1, month=MONTH_DEBUT_ANNEE_SCOLAIRE, day=1
    ) - datetime.timedelta(days=1)


def sem_decale_str(sem):
    """'D' si semestre decalé, ou ''"""
    # considère "décalé" les semestre impairs commençant entre janvier et juin
    # et les pairs entre juillet et decembre
    if sem["semestre_id"] <= 0:
        return ""
    if (sem["semestre_id"] % 2 and sem["mois_debut_ord"] <= 6) or (
        not sem["semestre_id"] % 2 and sem["mois_debut_ord"] > 6
    ):
        return "D"
    else:
        return ""


def is_valid_mail(email):
    """True if well-formed email address"""
    return re.match(r"^.+@.+\..{2,3}$", email)


def graph_from_edges(edges, graph_name="mygraph"):
    """Crée un graph pydot
    à partir d'une liste d'arêtes [ (n1, n2), (n2, n3), ... ]
    où n1, n2, ... sont des chaînes donnant l'id des nœuds.

    Fonction remplaçant celle de pydot qui est buggée.
    """
    nodes = set([it for tup in edges for it in tup])
    graph = pydot.Dot(graph_name)
    for n in nodes:
        graph.add_node(pydot.Node(n))
    for e in edges:
        graph.add_edge(pydot.Edge(src=e[0], dst=e[1]))
    return graph


ICONSIZES = {}  # name : (width, height) cache image sizes


def icontag(name, file_format="png", no_size=False, **attrs):
    """tag HTML pour un icone.
    (dans les versions anterieures on utilisait Zope)
    Les icones sont des fichiers PNG dans .../static/icons
    Si la taille (width et height) n'est pas spécifiée, lit l'image
    pour la mesurer (et cache le résultat).
    """
    if (not no_size) and (("width" not in attrs) or ("height" not in attrs)):
        if name not in ICONSIZES:
            img_file = os.path.join(
                Config.SCODOC_DIR,
                "app/static/icons/%s.%s"
                % (
                    name,
                    file_format,
                ),
            )
            with PILImage.open(img_file) as image:
                width, height = image.size[0], image.size[1]
            ICONSIZES[name] = (width, height)  # cache
        else:
            width, height = ICONSIZES[name]
        attrs["width"] = width
        attrs["height"] = height
    if "border" not in attrs:
        attrs["border"] = 0
    if "alt" not in attrs:
        attrs["alt"] = "logo %s" % name
    s = " ".join(['%s="%s"' % (k, attrs[k]) for k in attrs])
    return f'<img class="{name}" {s} src="{STATIC_DIR}/icons/{name}.{file_format}" />'


ICON_PDF = icontag("pdficon16x20_img", title="Version PDF")
ICON_XLS = icontag("xlsicon_img", title="Version tableur")

# HTML emojis
EMO_WARNING = "&#9888;&#65039;"  # warning /!\
EMO_RED_TRIANGLE_DOWN = "&#128315;"  # red triangle pointed down
EMO_PREV_ARROW = "&#10094;"
EMO_NEXT_ARROW = "&#10095;"


def sort_dates(L, reverse=False):
    """Return sorted list of dates, allowing None items (they are put at the beginning)"""
    mindate = datetime.datetime(datetime.MINYEAR, 1, 1)
    try:
        return sorted(L, key=lambda x: x or mindate, reverse=reverse)
    except:
        # Helps debugging
        log("sort_dates( %s )" % L)
        raise


def heterogeneous_sorting_key(x):
    "key to sort non homogeneous sequences"
    return (float(x), "") if isinstance(x, (bool, float, int)) else (-1e34, str(x))


def query_portal(req, msg="Portail Apogee", timeout=3):
    """Retreives external data using HTTP request
    (used to connect to Apogee portal, or ScoDoc server)
    returns a string,  "" on error
    """
    log("query_portal: %s" % req)
    error_message = None
    try:
        r = requests.get(req, timeout=timeout)  # seconds / request
    except requests.ConnectionError:
        error_message = "ConnectionError"
    except requests.Timeout:
        error_message = "Timeout"
    except requests.TooManyRedirects:
        error_message = "TooManyRedirects"
    except requests.RequestException:
        error_message = f"can't connect to {msg}"
    if error_message is not None:
        log(f"query_portal: {error_message}")
        return ""
    if r.status_code != 200:
        log(f"query_portal: http error {r.status_code}")
        return ""

    return r.text


def confirm_dialog(
    message="<p>Confirmer ?</p>",
    OK="OK",
    add_headers=True,  # complete page
    cancel_label="Annuler",
    cancel_url="",
    dest_url="",
    help_msg=None,
    parameters: dict = None,
    target_variable="dialog_confirmed",
):
    """HTML confirmation dialog: submit (POST) to same page or dest_url if given."""
    from app.scodoc import html_sco_header

    parameters = parameters or {}
    # dialog de confirmation simple
    parameters[target_variable] = 1
    # Attention: la page a pu etre servie en GET avec des parametres
    # si on laisse l'url "action" vide, les parametres restent alors que l'on passe en POST...
    if not dest_url:
        action = ""
    else:
        # strip remaining parameters from destination url:
        dest_url = urllib.parse.splitquery(dest_url)[0]
        action = f'action="{dest_url}"'

    H = [
        f"""<form {action} method="POST">
        {message}
        """,
    ]
    if OK or not cancel_url:
        H.append(f'<input type="submit" value="{OK}"/>')
    if cancel_url:
        H.append(
            f"""<input type ="button" value="{cancel_label}"
            onClick="document.location='{cancel_url}';"/>"""
        )
    for param in parameters.keys():
        if parameters[param] is None:
            parameters[param] = ""
        if isinstance(parameters[param], list):
            for e in parameters[param]:
                H.append(f"""<input type="hidden" name="{param}" value="{e}"/>""")
        else:
            H.append(
                f"""<input type="hidden" name="{param}" value="{parameters[param]}"/>"""
            )
    H.append("</form>")
    if help_msg:
        H.append('<p class="help">' + help_msg + "</p>")
    if add_headers:
        return (
            html_sco_header.sco_header() + "\n".join(H) + html_sco_header.sco_footer()
        )
    else:
        return "\n".join(H)


def objects_renumber(db, obj_list) -> None:
    """fixe les numeros des objets d'une liste de modèles
    pour ne pas changer son ordre"""
    log(f"objects_renumber")
    for i, obj in enumerate(obj_list):
        obj.numero = i
        db.session.add(obj)
    db.session.commit()


def comp_ranks(T: list[tuple]) -> dict[int, str]:
    """Calcul rangs à partir d'une liste ordonnée de tuples [ (valeur, ..., etudid) ]
    (valeur est une note numérique), en tenant compte des ex-aequos
    Le resultat est: { etudid : rang } où rang est une chaine decrivant le rang
    """
    rangs = {}  # { etudid : rang } (rang est une chaine)
    nb_ex = 0  # nb d'ex-aequo consécutifs en cours
    for i in range(len(T)):
        # test ex-aequo
        if i < len(T) - 1:
            next = T[i + 1][0]
        else:
            next = None
        moy = T[i][0]
        if nb_ex:
            srang = "%d ex" % (i + 1 - nb_ex)
            if moy == next:
                nb_ex += 1
            else:
                nb_ex = 0
        else:
            if moy == next:
                srang = "%d ex" % (i + 1 - nb_ex)
                nb_ex = 1
            else:
                srang = "%d" % (i + 1)
        rangs[T[i][-1]] = srang  # str(i+1)
    return rangs


def gen_cell(key: str, row: dict, elt="td", with_col_class=False):
    "html table cell"
    klass = row.get(f"_{key}_class", "")
    if with_col_class:
        klass = key + " " + klass
    attrs = f'class="{klass}"' if klass else ""
    if elt == "th":
        attrs += ' scope="row"'
    data = row.get(f"_{key}_data")  # dict
    if data:
        for k in data:
            attrs += f' data-{k}="{data[k]}"'
    order = row.get(f"_{key}_order")
    if order:
        attrs += f' data-order="{order}"'
    content = row.get(key, "")
    target = row.get(f"_{key}_target")
    target_attrs = row.get(f"_{key}_target_attrs", "")
    if target or target_attrs:  # avec lien
        href = f'href="{target}"' if target else ""
        content = f"<a {href} {target_attrs}>{content}</a>"
    return f"<{elt} {attrs}>{content}</{elt}>"


def gen_row(
    keys: list[str], row, elt="td", selected_etudid=None, with_col_classes=False
):
    "html table row"
    klass = row.get("_tr_class")
    if row.get("etudid", "") == selected_etudid:
        klass += " row_selected"
    tr_class = f'class="{klass}"' if klass else ""
    return f"""<tr {tr_class}>{
        "".join([gen_cell(key, row, elt, with_col_class=with_col_classes)
        for key in keys if not key.startswith('_')])
        }</tr>"""


# Pour accès depuis les templates jinja
def is_entreprises_enabled():
    from app.models import ScoDocSiteConfig

    return ScoDocSiteConfig.is_entreprises_enabled()