ScoDoc-Lille/app/scodoc/sco_apogee_compare.py

# -*- mode: python -*-
# -*- coding: utf-8 -*-

##############################################################################
#
# Gestion scolarite IUT
#
# Copyright (c) 1999 - 2024 Emmanuel Viennet.  All rights reserved.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
#   Emmanuel Viennet      emmanuel.viennet@viennet.net
#
##############################################################################

"""Comparaison de deux fichiers Apogée (maquettes)

1) Vérifier:
etape_apogee, vdi_apogee, cod_dip_apogee, annee_scolaire
structure: col_ids (la comparaison portera sur les colonnes communes)


2) Comparer listes d'étudiants
 Présents dans A mais pas dans B
 Présents dans B mais pas dans A
 nombre communs

3) Comparer résultats
Pour chaque étudiant commun:
 Pour chaque colonne commune:
    comparer les résultats

"""
from flask import g, url_for

from app import log
from app.scodoc import sco_apogee_csv, sco_apogee_reader
from app.scodoc.sco_apogee_csv import ApoData
from app.scodoc.gen_tables import GenTable
from app.scodoc.sco_exceptions import ScoValueError
from app.scodoc import html_sco_header
from app.scodoc import sco_preferences

_HELP_TXT = """
<div class="help">
<p>Outil de comparaison de fichiers (maquettes CSV) Apogée.
</p>
<p>Cet outil compare deux fichiers fournis. Aucune donnée stockée dans ScoDoc n'est utilisée.
</p>
</div>
"""


def apo_compare_csv_form():
    """Form: submit 2 CSV files to compare them."""
    H = [
        html_sco_header.sco_header(page_title="Comparaison de fichiers Apogée"),
        """<h2>Comparaison de fichiers Apogée</h2>
        <form id="apo_csv_add" action="apo_compare_csv" method="post" enctype="multipart/form-data">
        """,
        _HELP_TXT,
        """
        <div class="apo_compare_csv_form_but">
        Fichier Apogée A:
        <input type="file" size="30" name="file_a"/>
        </div>
        <div class="apo_compare_csv_form_but">
        Fichier Apogée B:
        <input type="file" size="30" name="file_b"/>
        </div>
        <input type="checkbox" name="autodetect" checked/>autodétecter encodage</input>
        <div class="apo_compare_csv_form_submit">
        <input type="submit" value="Comparer ces fichiers"/>
        </div>
        </form>""",
        html_sco_header.sco_footer(),
    ]
    return "\n".join(H)


def apo_compare_csv(file_a, file_b, autodetect=True):
    """Page comparing 2 Apogee CSV files"""
    try:
        apo_data_a = _load_apo_data(file_a, autodetect=autodetect)
        apo_data_b = _load_apo_data(file_b, autodetect=autodetect)
    except (UnicodeDecodeError, UnicodeEncodeError) as exc:
        dest_url = url_for("notes.semset_page", scodoc_dept=g.scodoc_dept)
        if autodetect:
            raise ScoValueError(
                """
            Erreur: l'encodage de l'un des fichiers est mal détecté.
            Essayez sans auto-détection, ou vérifiez le codage et le contenu
            des fichiers.
            """,
                dest_url=dest_url,
            ) from exc
        else:
            raise ScoValueError(
                f"""
            Erreur: l'encodage de l'un des fichiers est incorrect.
            Vérifiez qu'il est bien en {sco_apogee_reader.APO_INPUT_ENCODING}
            """,
                dest_url=dest_url,
            ) from exc
    H = [
        html_sco_header.sco_header(page_title="Comparaison de fichiers Apogée"),
        "<h2>Comparaison de fichiers Apogée</h2>",
        _HELP_TXT,
        '<div class="apo_compare_csv">',
        _apo_compare_csv(apo_data_a, apo_data_b),
        "</div>",
        """<p><a href="apo_compare_csv_form" class="stdlink">Autre comparaison</a></p>""",
        html_sco_header.sco_footer(),
    ]
    return "\n".join(H)


def _load_apo_data(csvfile, autodetect=True):
    "Read data from request variable and build ApoData"
    data_b = csvfile.read()
    if autodetect:
        data_b, message = sco_apogee_reader.fix_data_encoding(data_b)
        if message:
            log(f"apo_compare_csv: {message}")
        if not data_b:
            raise ScoValueError("fichier vide ? (apo_compare_csv: no data)")
    data = data_b.decode(sco_apogee_reader.APO_INPUT_ENCODING)
    apo_data = sco_apogee_csv.ApoData(data, orig_filename=csvfile.filename)
    return apo_data


def _apo_compare_csv(apo_a: ApoData, apo_b: ApoData):
    """Generate html report comparing A and B, two instances of ApoData
    representing Apogee CSV maquettes.
    """
    L = []
    # 1-- Check etape and codes
    L.append('<div class="section"><div class="tit">En-tête</div>')
    L.append('<div><span class="key">Nom fichier A:</span><span class="val_ok">')
    L.append(apo_a.orig_filename)
    L.append("</span></div>")
    L.append('<div><span class="key">Nom fichier B:</span><span class="val_ok">')
    L.append(apo_b.orig_filename)
    L.append("</span></div>")
    L.append('<div><span class="key">Étape Apogée:</span>')
    if apo_a.etape_apogee != apo_b.etape_apogee:
        L.append(
            f"""<span class="val_dif">{apo_a.etape_apogee} != {apo_b.etape_apogee}</span>"""
        )
    else:
        L.append(f"""<span class="val_ok">{apo_a.etape_apogee}</span>""")
    L.append("</div>")

    L.append('<div><span class="key">VDI Apogée:</span>')
    if apo_a.vdi_apogee != apo_b.vdi_apogee:
        L.append(
            f"""<span class="val_dif">{apo_a.vdi_apogee} != {apo_b.vdi_apogee}</span>"""
        )
    else:
        L.append(f"""<span class="val_ok">{apo_a.vdi_apogee}</span>""")
    L.append("</div>")

    L.append('<div><span class="key">Code diplôme :</span>')
    if apo_a.cod_dip_apogee != apo_b.cod_dip_apogee:
        L.append(
            f"""<span class="val_dif">{apo_a.cod_dip_apogee} != {apo_b.cod_dip_apogee}</span>"""
        )
    else:
        L.append(f"""<span class="val_ok">{apo_a.cod_dip_apogee}</span>""")
    L.append("</div>")

    L.append('<div><span class="key">Année scolaire :</span>')
    if apo_a.annee_scolaire != apo_b.annee_scolaire:
        L.append(
            '<span class="val_dif">%s != %s</span>'
            % (apo_a.annee_scolaire, apo_b.annee_scolaire)
        )
    else:
        L.append('<span class="val_ok">%s</span>' % (apo_a.annee_scolaire,))
    L.append("</div>")

    # Colonnes:
    a_elts = set(apo_a.apo_csv.apo_elts.keys())
    b_elts = set(apo_b.apo_csv.apo_elts.keys())
    L.append('<div><span class="key">Éléments Apogée :</span>')
    if a_elts == b_elts:
        L.append(f"""<span class="val_ok">{len(a_elts)}</span>""")
    else:
        elts_communs = a_elts.intersection(b_elts)
        elts_only_a = a_elts - a_elts.intersection(b_elts)
        elts_only_b = b_elts - a_elts.intersection(b_elts)
        L.append(
            '<span class="val_dif">différents (%d en commun, %d seulement dans A, %d seulement dans B)</span>'
            % (
                len(elts_communs),
                len(elts_only_a),
                len(elts_only_b),
            )
        )
        if elts_only_a:
            L.append(
                '<div span class="key">Éléments seulement dans A : </span><span class="val_dif">%s</span></div>'
                % ", ".join(sorted(elts_only_a))
            )
        if elts_only_b:
            L.append(
                '<div span class="key">Éléments seulement dans B : </span><span class="val_dif">%s</span></div>'
                % ", ".join(sorted(elts_only_b))
            )
    L.append("</div>")
    L.append("</div>")  # /section

    # 2--
    L.append('<div class="section"><div class="tit">Étudiants</div>')

    a_nips = set(apo_a.etud_by_nip)
    b_nips = set(apo_b.etud_by_nip)
    nb_etuds_communs = len(a_nips.intersection(b_nips))
    nb_etuds_dif = len(a_nips.union(b_nips) - a_nips.intersection(b_nips))
    L.append("""<div><span class="key">Liste d'étudiants :</span>""")
    if a_nips == b_nips:
        L.append(
            f"""<span class="s_ok">
        {len(a_nips)} étudiants (tous présents dans chaque fichier)</span>
        """
        )
    else:
        L.append(
            f"""<span class="val_dif">différents ({nb_etuds_communs} en commun, {
                    nb_etuds_dif} différents)</span>"""
        )
    L.append("</div>")
    L.append("</div>")  # /section

    # 3-- Résultats de chaque étudiant:
    if nb_etuds_communs > 0:
        L.append(
            """<div class="section sec_table">
        <div class="tit">Différences de résultats des étudiants présents dans les deux fichiers
        </div>
        <p>
        """
        )
        T = apo_table_compare_etud_results(apo_a, apo_b)
        if T.get_nb_rows() > 0:
            L.append(T.html())
        else:
            L.append(
                f"""<p class="p_ok">aucune différence de résultats
            sur les {nb_etuds_communs} étudiants communs
            (<em>les éléments Apogée n'apparaissant pas dans les deux
            fichiers sont omis</em>)
            </p>
            """
            )
        L.append("</div>")  # /section

    return "\n".join(L)


def apo_table_compare_etud_results(A, B):
    """"""
    D = compare_etuds_res(A, B)
    T = GenTable(
        rows=D,
        titles={
            "nip": "NIP",
            "nom": "Nom",
            "prenom": "Prénom",
            "elt_code": "Element",
            "type_res": "Type",
            "val_A": "A: %s" % (A.orig_filename or ""),
            "val_B": "B: %s" % (B.orig_filename or ""),
        },
        columns_ids=("nip", "nom", "prenom", "elt_code", "type_res", "val_A", "val_B"),
        html_class="table_leftalign",
        html_with_td_classes=True,
        preferences=sco_preferences.SemPreferences(),
        table_id="apo_table_compare_etud_results",
    )
    return T


def _build_etud_res(e, apo_data):
    r = {}
    for elt_code in apo_data.apo_csv.apo_elts:
        elt = apo_data.apo_csv.apo_elts[elt_code]
        try:
            # les colonnes de cet élément
            col_ids_type = [(ec["apoL_a01_code"], ec["Type Rés."]) for ec in elt.cols]
        except KeyError as exc:
            raise ScoValueError(
                "Erreur: un élément sans 'Type Rés.'. Vérifiez l'encodage de vos fichiers."
            ) from exc
        r[elt_code] = {}
        for col_id, type_res in col_ids_type:
            r[elt_code][type_res] = e.cols[col_id]
    return r


def compare_etud_res(r_A, r_B, remove_missing=True):
    """Pour chaque valeur difference dans les resultats d'un etudiant
    elt_code   type_res   val_A     val_B
    """
    diffs = []
    elt_codes = set(r_A).union(set(r_B))
    for elt_code in elt_codes:
        for type_res in r_A.get(elt_code, r_B.get(elt_code)):
            if elt_code not in r_A:
                if remove_missing:
                    continue
                else:
                    val_A = None  # element absent
            else:
                val_A = r_A[elt_code][type_res]
            if elt_code not in r_B:
                if remove_missing:
                    continue
                else:
                    val_B = None  # element absent
            else:
                val_B = r_B[elt_code][type_res]
            if type_res == "N":
                # Cas particulier pour les notes: compare les nombres
                try:
                    val_A_num = float(val_A.replace(",", "."))
                    val_B_num = float(val_B.replace(",", "."))
                except ValueError:
                    val_A_num, val_B_num = val_A, val_B
                val_A, val_B = val_A_num, val_B_num
            if val_A != val_B:
                diffs.append(
                    {
                        "elt_code": elt_code,
                        "type_res": type_res,
                        "val_A": val_A,
                        "val_B": val_B,
                    }
                )
    return diffs


def compare_etuds_res(A, B):
    """
    nip, nom, prenom, elt_code, type_res, val_A, val_B
    """
    A_nips = set(A.etud_by_nip)
    B_nips = set(B.etud_by_nip)
    common_nips = A_nips.intersection(B_nips)
    # A_not_B_nips = A_nips - B_nips
    # B_not_A_nips = B_nips - A_nips
    D = []
    for nip in common_nips:
        etu_A = A.etud_by_nip[nip]
        etu_B = B.etud_by_nip[nip]
        r_A = _build_etud_res(etu_A, A)
        r_B = _build_etud_res(etu_B, B)
        diffs = compare_etud_res(r_A, r_B)
        for d in diffs:
            d.update(
                {"nip": etu_A["nip"], "nom": etu_A["nom"], "prenom": etu_A["prenom"]}
            )
            D.append(d)
    return D