Amélioration sco_excel

app/entreprises/routes.py

@@ -1419,7 +1419,7 @@ def get_import_donnees_file_sample():
 @permission_required(Permission.RelationsEntreprisesExport)
 def import_donnees():
     """
-    Permet d'importer des entreprises a l'aide d'un fichier excel (.xlsx)
+    Permet d'importer des entreprises à partir d'un fichier excel (.xlsx)
     """
     form = ImportForm()
     if form.validate_on_submit():
@@ -1428,7 +1428,7 @@ def import_donnees():
             Config.SCODOC_VAR_DIR, "tmp", secure_filename(file.filename)
         )
         file.save(file_path)
-        diag, lm = sco_excel.excel_file_to_list_are(file_path)
+        diag, lm = sco_excel.excel_workbook_to_list(file_path)
         os.remove(file_path)
         if lm is None or len(lm) < 2:
             flash("Veuillez utilisez la feuille excel à remplir")

app/scodoc/sco_excel.py

@@ -40,10 +40,9 @@ from openpyxl.comments import Comment
 from openpyxl import Workbook, load_workbook
 from openpyxl.cell import WriteOnlyCell
 from openpyxl.styles import Font, Border, Side, Alignment, PatternFill
+from openpyxl.worksheet.worksheet import Worksheet
 
 import app.scodoc.sco_utils as scu
-from app.scodoc import notesdb
-from app.scodoc import sco_preferences
 from app import log
 from app.scodoc.sco_exceptions import ScoValueError
 
@@ -593,71 +592,87 @@ def excel_feuille_saisie(e, titreannee, description, lines):
 def excel_bytes_to_list(bytes_content):
     try:
         filelike = io.BytesIO(bytes_content)
-        return _excel_to_list(filelike)
+    except Exception as exc:
-    except:
         raise ScoValueError(
             """Le fichier xlsx attendu n'est pas lisible !
             Peut-être avez-vous fourni un fichier au mauvais format (txt, xls, ..)
             """
-        )
+        ) from exc
+    return _excel_to_list(filelike)
 
 
 def excel_file_to_list(filename):
     try:
         return _excel_to_list(filename)
-    except:
+    except Exception as exc:
         raise ScoValueError(
             """Le fichier xlsx attendu n'est pas lisible !
             Peut-être avez-vous fourni un fichier au mauvais format (txt, xls, ...)
             """
-        )
+        ) from exc
 
 
 def excel_file_to_list_are(filename):
     try:
         return _excel_to_list_are(filename)
-    except:
+    except Exception as exc:
         raise ScoValueError(
             """Le fichier xlsx attendu n'est pas lisible !
             Peut-être avez-vous fourni un fichier au mauvais format (txt, xls, ...)
             """
-        )
+        ) from exc
+
+
+def _open_workbook(filelike, dump_debug=False) -> Workbook:
+    """Open document.
+    On error, if dump-debug is True, dump data in /tmp for debugging purpose
+    """
+    try:
+        workbook = load_workbook(filename=filelike, read_only=True, data_only=True)
+    except Exception as exc:
+        log("Excel_to_list: failure to import document")
+        if dump_debug:
+            dump_filename = "/tmp/last_scodoc_import_failure" + scu.XLSX_SUFFIX
+            log(f"Dumping problemetic file on {dump_filename}")
+            with open(dump_filename, "wb") as f:
+                f.write(filelike)
+        raise ScoValueError(
+            "Fichier illisible: assurez-vous qu'il s'agit bien d'un document Excel xlsx !"
+        ) from exc
+    return workbook
 
 
 def _excel_to_list(filelike):
-    """returns list of list
+    """returns list of list"""
-    convert_to_string is a conversion function applied to all non-string values (ie numbers)
+    workbook = _open_workbook(filelike)
-    """
-    try:
-        wb = load_workbook(filename=filelike, read_only=True, data_only=True)
-    except:
-        log("Excel_to_list: failure to import document")
-        with open("/tmp/last_scodoc_import_failure" + scu.XLSX_SUFFIX, "wb") as f:
-            f.write(filelike)
-        raise ScoValueError(
-            "Fichier illisible: assurez-vous qu'il s'agit bien d'un document Excel !"
-        )
     diag = []  # liste de chaines pour former message d'erreur
-    # n'utilise que la première feuille
+    if len(workbook.get_sheet_names()) < 1:
-    if len(wb.get_sheet_names()) < 1:
         diag.append("Aucune feuille trouvée dans le classeur !")
         return diag, None
-    if len(wb.get_sheet_names()) > 1:
+    # n'utilise que la première feuille:
+    if len(workbook.get_sheet_names()) > 1:
         diag.append("Attention: n'utilise que la première feuille du classeur !")
+    sheet_name = workbook.get_sheet_names()[0]
+    ws = workbook[sheet_name]
+    matrix, diag_sheet = _excel_sheet_to_list(ws, sheet_name)
+    diag += diag_sheet
+    return diag, matrix
+
+
+def _excel_sheet_to_list(sheet: Worksheet, sheet_name: str) -> tuple[list, list]:
+    """read a spreadsheet sheet, and returns:
+    - diag : a list of strings (error messages aimed at helping the user)
+    - a list of lists: the spreadsheet cells
+    """
+    diag = []
     # fill matrix
-    sheet_name = wb.get_sheet_names()[0]
-    ws = wb.get_sheet_by_name(sheet_name)
-    sheet_name = sheet_name.encode(scu.SCO_ENCODING, "backslashreplace")
     values = {}
-    for row in ws.iter_rows():
+    for row in sheet.iter_rows():
         for cell in row:
             if cell.value is not None:
                 values[(cell.row - 1, cell.column - 1)] = str(cell.value)
     if not values:
-        diag.append(
+        diag.append(f"Aucune valeur trouvée dans la feuille {sheet_name} !")
-            "Aucune valeur trouvée dans la feuille %s !"
-            % sheet_name.decode(scu.SCO_ENCODING)
-        )
         return diag, None
     indexes = list(values.keys())
     # search numbers of rows and cols
@@ -665,76 +680,38 @@ def _excel_to_list(filelike):
     cols = [x[1] for x in indexes]
     nbcols = max(cols) + 1
     nbrows = max(rows) + 1
-    m = []
+    matrix = []
     for _ in range(nbrows):
-        m.append([""] * nbcols)
+        matrix.append([""] * nbcols)
 
     for row_idx, col_idx in indexes:
         v = values[(row_idx, col_idx)]
-        # if isinstance(v, six.text_type):
+        matrix[row_idx][col_idx] = v
-        #     v = v.encode(scu.SCO_ENCODING, "backslashreplace")
+    diag.append(f'Feuille "{sheet_name}", {len(matrix)} lignes')
-        # elif convert_to_string:
+
-        #     v = convert_to_string(v)
+    return diag, matrix
-        m[row_idx][col_idx] = v
-    diag.append(
-        'Feuille "%s", %d lignes' % (sheet_name.decode(scu.SCO_ENCODING), len(m))
-    )
-    # diag.append(str(M))
-    #
-    return diag, m
 
 
-def _excel_to_list_are(filelike):
+def _excel_workbook_to_list(filelike):
-    """returns list of list
+    """Lit un classeur (workbook): chaque feuille est lue
-    convert_to_string is a conversion function applied to all non-string values (ie numbers)
+        et est convertie en une liste de listes.
+    Returns:
+    - diag : a list of strings (error messages aimed at helping the user)
+    - a list of lists: the spreadsheet cells
     """
-    try:
+    workbook = _open_workbook(filelike)
-        wb = load_workbook(filename=filelike, read_only=True, data_only=True)
-    except:
-        log("Excel_to_list: failure to import document")
-        with open("/tmp/last_scodoc_import_failure" + scu.XLSX_SUFFIX, "wb") as f:
-            f.write(filelike)
-        raise ScoValueError(
-            "Fichier illisible: assurez-vous qu'il s'agit bien d'un document Excel !"
-        )
     diag = []  # liste de chaines pour former message d'erreur
-    if len(wb.get_sheet_names()) < 1:
+    if len(workbook.get_sheet_names()) < 1:
         diag.append("Aucune feuille trouvée dans le classeur !")
         return diag, None
-    lm = []
+    matrix_list = []
-    for sheet_name in wb.get_sheet_names():
+    for sheet_name in workbook.get_sheet_names():
         # fill matrix
-        ws = wb.get_sheet_by_name(sheet_name)
+        sheet = workbook.get_sheet_by_name(sheet_name)
-        sheet_name = sheet_name.encode(scu.SCO_ENCODING, "backslashreplace")
+        matrix, diag_sheet = _excel_sheet_to_list(sheet, sheet_name)
-        values = {}
+        diag += diag_sheet
-        for row in ws.iter_rows():
+        matrix_list.append(matrix)
-            for cell in row:
+    return diag, matrix_list
-                if cell.value is not None:
-                    values[(cell.row - 1, cell.column - 1)] = str(cell.value)
-        if not values:
-            diag.append(
-                "Aucune valeur trouvée dans la feuille %s !"
-                % sheet_name.decode(scu.SCO_ENCODING)
-            )
-            return diag, None
-        indexes = list(values.keys())
-        # search numbers of rows and cols
-        rows = [x[0] for x in indexes]
-        cols = [x[1] for x in indexes]
-        nbcols = max(cols) + 1
-        nbrows = max(rows) + 1
-        m = []
-        for _ in range(nbrows):
-            m.append([""] * nbcols)
-
-        for row_idx, col_idx in indexes:
-            v = values[(row_idx, col_idx)]
-            m[row_idx][col_idx] = v
-        diag.append(
-            'Feuille "%s", %d lignes' % (sheet_name.decode(scu.SCO_ENCODING), len(m))
-        )
-        lm.append(m)
-    return diag, lm
 
 
 def excel_feuille_listeappel(