Merge pull request 'improve_csv_read' (#529) from jmplace/ScoDoc-Lille:improve_csv_read into master

Reviewed-on: https://scodoc.org/git/ScoDoc/ScoDoc/pulls/529

tests/api/make_samples.py

@@ -40,16 +40,19 @@ Le tout mis en forme au format markdown et rangé dans le répertoire DATA_DIR (
 TODO: ajouter un argument au script permettant de ne générer qu'un seul fichier (exemple: `python make_samples.py nom_exemple`)
 
 """
+import numpy as np
 import os
 import shutil
 import sys
 import re
 from collections import defaultdict
 from pprint import pprint as pp
-from pprint import pformat as pf
 
 import urllib3
 import json
+
+from pandas import read_csv
+
 from setup_test_api import (
     API_PASSWORD,
     API_URL,
@@ -63,6 +66,7 @@ from setup_test_api import (
 )
 
 DATA_DIR = "/tmp/samples/"
+SAMPLES_FILENAME = "tests/ressources/samples.csv"
 
 
 class Sample:
@@ -143,7 +147,12 @@ class Samples:
         self.entries = defaultdict(lambda: set())
         self.entry_names = entry_names
 
-    def add_sample(self, entry, url, method="GET", permission="ScoView", content=None):
+    def add_sample(self, line):
+        entry = line["entry_name"]
+        url = line["url"]
+        method = line["method"]
+        permission = line["permission"]
+        content = line["content"]
         if self.entry_names is None or entry in self.entry_names:
             if method[0] == "#":
                 detail = "**ignored**"
@@ -185,19 +194,24 @@ def make_samples():
             shutil.rmtree(DATA_DIR)
             os.mkdir(DATA_DIR)
     else:
-        os.mkdir("/tmp/samples")
+        os.mkdir(DATA_DIR)
 
     samples = Samples(entry_names)
-    samples_file = os.path.dirname(__file__) + "/samples.csv"
+    df = read_csv(
-    with open(samples_file) as f:
+        SAMPLES_FILENAME,
-        L = [x[:-1].split("\t") for x in f]
+        sep=";",
-    for line in L[1:]:
+        quotechar='"',
-        entry_name = line[0]
+        dtype={
-        url = line[1]
+            "entry_name": str,
-        permission = line[2] if line[2] != "" else "ScoView"
+            "url": str,
-        method = line[3] if line[3] != "" else "GET"
+            "permission": str,
-        content = line[4] if len(line) > 4 else ""
+            "method": str,
-        samples.add_sample(entry_name, url, method, permission, content)
+            "content": str,
+        },
+        keep_default_na=False,
+    )
+    df = df.reset_index()
+    df.apply(lambda line: samples.add_sample(line), axis=1)
     samples.dump()
     return samples
 

tests/api/samples.csv

@@ -1,99 +0,0 @@
-reference	url	permission	method	content
-departements	/departements		GET	
-departements-ids	/departements_ids		GET	
-departement	/departement/TAPI		GET	
-departement	/departement/id/1		GET	
-departement-etudiants	/departement/TAPI/etudiants		GET	
-departement-etudiants	/departement/id/1/etudiants		GET	
-departement-formsemestres_ids	/departement/TAPI/formsemestres_ids		GET	
-departement-formsemestres_ids	/departement/id/1/formsemestres_ids		GET	
-departement-formsemestres-courants	/departement/TAPI/formsemestres_courants		GET	
-departement-formsemestres-courants	/departement/id/1/formsemestres_courants		GET	
-departement-create	/departement/create	ScoSuperAdmin	POST	{"acronym": "NEWONE" , "visible": true}
-departement-edit	/departement/NEWONE/edit	ScoSuperAdmin	POST	{"visible": false}
-departement-delete	/departement/NEWONE/delete	ScoSuperAdmin	POST	
-etudiants-courants	/etudiants/courants?date_courante=2022-07-20	ScoView	GET
-etudiants-courants	/etudiants/courants/long?date_courante=2022-07-20	ScoView	GET
-etudiant	/etudiant/etudid/11	ScoView	GET
-etudiant	/etudiant/nip/11	ScoView	GET
-etudiant	/etudiant/ine/INE11	ScoView	GET
-etudiants-clef	/etudiants/etudid/11	ScoView	GET
-etudiants-clef	/etudiants/ine/INE11	ScoView	GET
-etudiants-clef	/etudiants/nip/11	ScoView	GET
-etudiant-formsemestres	/etudiant/etudid/11/formsemestres	ScoView	GET
-etudiant-formsemestres	/etudiant/ine/INE11/formsemestres	ScoView	GET
-etudiant_formsemestres	/etudiant/nip/11/formsemestres	ScoView	GET
-etudiant-formsemestre-bulletin	/etudiant/etudid/11/formsemestre/1/bulletin	ScoView	GET
-etudiant-formsemestre-bulletin	/etudiant/ine/INE11/formsemestre/1/bulletin	ScoView	GET
-etudiant-formsemestre-bulletin	/etudiant/nip/11/formsemestre/1/bulletin	ScoView	GET
-etudiant-formsemestre-bulletin	/etudiant/nip/11/formsemestre/1/bulletin/short/pdf	ScoView	GET
-etudiant-formsemestre-groups	/etudiant/etudid/11/formsemestre/1/groups	ScoView	GET
-formations	/formations	ScoView	GET
-formations_ids	/formations_ids	ScoView	GET
-formation	/formation/1	ScoView	GET
-formation-export	/formation/1/export	ScoView	GET
-formation-export	/formation/1/export_with_ids	ScoView	GET
-formation-referentiel_competences	/formation/1/referentiel_competences	ScoView	GET
-moduleimpl	/moduleimpl/1	ScoView	GET
-formsemestre	/formsemestre/1	ScoView	GET
-formsemestres-query	/formsemestres/query?annee_scolaire=2022&etape_apo=A2	ScoView	GET
-formsemestres-query	/formsemestres/query?nip=11	ScoView	GET
-formsemestre-bulletins	/formsemestre/1/bulletins	ScoView	GET
-formsemestre-programme	/formsemestre/1/programme	ScoView	GET
-formsemestre-etudiants	/formsemestre/1/etudiants	ScoView	GET
-formsemestre-etudiants	/formsemestre/1/etudiants/long	ScoView	GET
-formsemestre-etudiants-query	/formsemestre/1/etudiants/query?etat=D	ScoView	GET
-formsemestre-etat_evals	/formsemestre/1/etat_evals	ScoView	GET
-formsemestre-resultats	/formsemestre/1/resultats	ScoView	GET
-formsemestre-decisions_jury	/formsemestre/1/decisions_jury	ScoView	GET
-formsemestre-partitions	/formsemestre/1/partitions	ScoView	GET
-partition	/partition/1	ScoView	GET
-group-etudiants	/group/1/etudiants	ScoView	GET
-group-etudiants-query	/group/1/etudiants/query?etat=D	ScoView	GET
-moduleimpl-evaluations	/moduleimpl/1/evaluations	ScoView	GET
-evaluation-notes	/evaluation/1/notes	ScoView	GET
-user	/user/1	ScoView	GET
-users-query	/users/query?starts_with=u_	ScoView	GET
-permissions	/permissions	ScoView	GET
-roles	/roles	ScoView	GET
-role	/role/Observateur	ScoView	GET
-group-set_etudiant	/group/1/set_etudiant/10	ScoSuperAdmin	POST	
-group-remove_etudiant	/group/1/remove_etudiant/10	ScoSuperAdmin	POST	
-partition-group-create	/partition/1/group/create	ScoSuperAdmin	POST	{"group_name": "NEW_GROUP"}
-group-edit	/group/2/edit	ScoSuperAdmin	POST	{"group_name": "NEW_GROUP2"}
-group-delete	/group/2/delete	ScoSuperAdmin	POST	
-formsemestre-partition-create	/formsemestre/1/partition/create	ScoSuperAdmin	POST	{"partition_name": "PART"} 
-formsemestre-partitions-order	/formsemestre/1/partitions/order	ScoSuperAdmin	POST	[ 1 ]
-partition-edit	/partition/1/edit	ScoSuperAdmin	POST	{"partition_name":"P2BIS", "numero":3,"bul_show_rank":true,"show_in_lists":false, "groups_editable":true}
-partition-remove_etudiant	/partition/2/remove_etudiant/10	ScoSuperAdmin	POST	
-partition-groups-order	/partition/1/groups/order	ScoSuperAdmin	POST	[ 1 ]
-partition-delete	/partition/2/delete	ScoSuperAdmin	POST	
-user-create	/user/create	ScoSuperAdmin	POST	{"user_name": "alain", "dept": null, "nom": "alain", "prenom": "bruno", "active": true }
-user-edit	/user/10/edit	ScoSuperAdmin	POST	{ "dept":  "TAPI", "nom": "alain2", "prenom": "bruno2", "active": false }
-user-password	/user/3/password	ScoSuperAdmin	POST	{ "password": "rePlaCemeNT456averylongandcomplicated" }
-user-password	/user/3/password	ScoSuperAdmin	POST	{ "password": "too_simple" }
-user-role-add	/user/10/role/Observateur/add	ScoSuperAdmin	POST	
-user-role-remove	/user/10/role/Observateur/remove	ScoSuperAdmin	POST	
-role-create	/role/create/customRole	ScoSuperAdmin	POST	{"permissions": ["ScoView", "ScoUsersView"]}
-role-remove_permission	/role/customRole/remove_permission/ScoUsersView	ScoSuperAdmin	POST	
-role-add_permission	/role/customRole/add_permission/ScoUsersView	ScoSuperAdmin	POST	
-role-edit	/role/customRole/edit	ScoSuperAdmin	POST	{ "name" : "LaveurDeVitres", "permissions" : [ "ScoView" ] }
-role-delete	/role/customRole/delete	ScoSuperAdmin	POST	
-logos	/logos	ScoSuperAdmin	GET	
-logo	/logo/demo	ScoSuperAdmin	GET	
-departement-logos	/departement/TAPI/logos	ScoSuperAdmin	GET	
-departement-logos	/departement/id/1/logos	ScoSuperAdmin	GET	
-departement-logo	/departement/TAPI/logo/demo	ScoSuperAdmin	GET	
-departement-logo	/departement/id/1/logo/demo	ScoSuperAdmin	GET	
-test-pdf	/etudiant/nip/11/formsemestre/1/bulletin/pdf	ScoView	GET
-test-pdf	/etudiant/nip/11/formsemestre/1/bulletin/pdf	ScoView	GET
-test-pdf	/etudiant/etudid/11/formsemestre/1/bulletin/short/pdf	ScoView	GET
-test-pdf	/etudiant/ine/INE11/formsemestre/1/bulletin/short/pdf	ScoView	GET
-test-pdf	/etudiant/nip/11/formsemestre/1/bulletin/short/pdf	ScoView	GET
-test-pdf	/etudiant/etudid/11/formsemestre/1/bulletin/pdf	ScoView	GET
-test-pdf	/etudiant/etudid/11/formsemestre/1/bulletin/short	ScoView	GET
-test-pdf	/etudiant/ine/INE11/formsemestre/1/bulletin/short	ScoView	GET
-test-pdf	/etudiant/nip/11/formsemestre/1/bulletin/short	ScoView	GET
-test-pdf	/etudiant/etudid/11/formsemestre/1/bulletin	ScoView	GET
-test-pdf	/etudiant/ine/INE11/formsemestre/1/bulletin	ScoView	GET
-test-pdf	/etudiant/nip/11/formsemestre/1/bulletin	ScoView	GET

tests/ressources/samples.csv

@@ -0,0 +1,99 @@
+"entry_name";"url";"permission";"method";"content"
+"departements";"/departements";"ScoView";"GET";
+"departements-ids";"/departements_ids";"ScoView";"GET";
+"departement";"/departement/TAPI";"ScoView";"GET";
+"departement";"/departement/id/1";"ScoView";"GET";
+"departement-etudiants";"/departement/TAPI/etudiants";"ScoView";"GET";
+"departement-etudiants";"/departement/id/1/etudiants";"ScoView";"GET";
+"departement-formsemestres_ids";"/departement/TAPI/formsemestres_ids";"ScoView";"GET";
+"departement-formsemestres_ids";"/departement/id/1/formsemestres_ids";"ScoView";"GET";
+"departement-formsemestres-courants";"/departement/TAPI/formsemestres_courants";"ScoView";"GET";
+"departement-formsemestres-courants";"/departement/id/1/formsemestres_courants";"ScoView";"GET";
+"departement-create";"/departement/create";"ScoSuperAdmin";"POST";"{""acronym"": ""NEWONE"" , ""visible"": true}"
+"departement-edit";"/departement/NEWONE/edit";"ScoSuperAdmin";"POST";"{""visible"": false}"
+"departement-delete";"/departement/NEWONE/delete";"ScoSuperAdmin";"POST";
+"etudiants-courants";"/etudiants/courants?date_courante=2022-07-20";"ScoView";"GET";
+"etudiants-courants";"/etudiants/courants/long?date_courante=2022-07-20";"ScoView";"GET";
+"etudiant";"/etudiant/etudid/11";"ScoView";"GET";
+"etudiant";"/etudiant/nip/11";"ScoView";"GET";
+"etudiant";"/etudiant/ine/INE11";"ScoView";"GET";
+"etudiants-clef";"/etudiants/etudid/11";"ScoView";"GET";
+"etudiants-clef";"/etudiants/ine/INE11";"ScoView";"GET";
+"etudiants-clef";"/etudiants/nip/11";"ScoView";"GET";
+"etudiant-formsemestres";"/etudiant/etudid/11/formsemestres";"ScoView";"GET";
+"etudiant-formsemestres";"/etudiant/ine/INE11/formsemestres";"ScoView";"GET";
+"etudiant_formsemestres";"/etudiant/nip/11/formsemestres";"ScoView";"GET";
+"etudiant-formsemestre-bulletin";"/etudiant/etudid/11/formsemestre/1/bulletin";"ScoView";"GET";
+"etudiant-formsemestre-bulletin";"/etudiant/ine/INE11/formsemestre/1/bulletin";"ScoView";"GET";
+"etudiant-formsemestre-bulletin";"/etudiant/nip/11/formsemestre/1/bulletin";"ScoView";"GET";
+"etudiant-formsemestre-bulletin";"/etudiant/nip/11/formsemestre/1/bulletin/short/pdf";"ScoView";"GET";
+"etudiant-formsemestre-groups";"/etudiant/etudid/11/formsemestre/1/groups";"ScoView";"GET";
+"formations";"/formations";"ScoView";"GET";
+"formations_ids";"/formations_ids";"ScoView";"GET";
+"formation";"/formation/1";"ScoView";"GET";
+"formation-export";"/formation/1/export";"ScoView";"GET";
+"formation-export";"/formation/1/export_with_ids";"ScoView";"GET";
+"formation-referentiel_competences";"/formation/1/referentiel_competences";"ScoView";"GET";
+"moduleimpl";"/moduleimpl/1";"ScoView";"GET";
+"formsemestre";"/formsemestre/1";"ScoView";"GET";
+"formsemestres-query";"/formsemestres/query?annee_scolaire=2022&etape_apo=A2";"ScoView";"GET";
+"formsemestres-query";"/formsemestres/query?nip=11";"ScoView";"GET";
+"formsemestre-bulletins";"/formsemestre/1/bulletins";"ScoView";"GET";
+"formsemestre-programme";"/formsemestre/1/programme";"ScoView";"GET";
+"formsemestre-etudiants";"/formsemestre/1/etudiants";"ScoView";"GET";
+"formsemestre-etudiants";"/formsemestre/1/etudiants/long";"ScoView";"GET";
+"formsemestre-etudiants-query";"/formsemestre/1/etudiants/query?etat=D";"ScoView";"GET";
+"formsemestre-etat_evals";"/formsemestre/1/etat_evals";"ScoView";"GET";
+"formsemestre-resultats";"/formsemestre/1/resultats";"ScoView";"GET";
+"formsemestre-decisions_jury";"/formsemestre/1/decisions_jury";"ScoView";"GET";
+"formsemestre-partitions";"/formsemestre/1/partitions";"ScoView";"GET";
+"partition";"/partition/1";"ScoView";"GET";
+"group-etudiants";"/group/1/etudiants";"ScoView";"GET";
+"group-etudiants-query";"/group/1/etudiants/query?etat=D";"ScoView";"GET";
+"moduleimpl-evaluations";"/moduleimpl/1/evaluations";"ScoView";"GET";
+"evaluation-notes";"/evaluation/1/notes";"ScoView";"GET";
+"user";"/user/1";"ScoView";"GET";
+"users-query";"/users/query?starts_with=u_";"ScoView";"GET";
+"permissions";"/permissions";"ScoView";"GET";
+"roles";"/roles";"ScoView";"GET";
+"role";"/role/Observateur";"ScoView";"GET";
+"group-set_etudiant";"/group/1/set_etudiant/10";"ScoSuperAdmin";"POST";
+"group-remove_etudiant";"/group/1/remove_etudiant/10";"ScoSuperAdmin";"POST";
+"partition-group-create";"/partition/1/group/create";"ScoSuperAdmin";"POST";"{""group_name"": ""NEW_GROUP""}"
+"group-edit";"/group/2/edit";"ScoSuperAdmin";"POST";"{""group_name"": ""NEW_GROUP2""}"
+"group-delete";"/group/2/delete";"ScoSuperAdmin";"POST";
+"formsemestre-partition-create";"/formsemestre/1/partition/create";"ScoSuperAdmin";"POST";"{""partition_name"": ""PART""} "
+"formsemestre-partitions-order";"/formsemestre/1/partitions/order";"ScoSuperAdmin";"POST";"[ 1 ]"
+"partition-edit";"/partition/1/edit";"ScoSuperAdmin";"POST";"{""partition_name"":""P2BIS"", ""numero"":3,""bul_show_rank"":true,""show_in_lists"":false, ""groups_editable"":true}"
+"partition-remove_etudiant";"/partition/2/remove_etudiant/10";"ScoSuperAdmin";"POST";
+"partition-groups-order";"/partition/1/groups/order";"ScoSuperAdmin";"POST";"[ 1 ]"
+"partition-delete";"/partition/2/delete";"ScoSuperAdmin";"POST";
+"user-create";"/user/create";"ScoSuperAdmin";"POST";"{""user_name"": ""alain"", ""dept"": null, ""nom"": ""alain"", ""prenom"": ""bruno"", ""active"": true }"
+"user-edit";"/user/10/edit";"ScoSuperAdmin";"POST";"{ ""dept"":  ""TAPI"", ""nom"": ""alain2"", ""prenom"": ""bruno2"", ""active"": false }"
+"user-password";"/user/3/password";"ScoSuperAdmin";"POST";"{ ""password"": ""rePlaCemeNT456averylongandcomplicated"" }"
+"user-password";"/user/3/password";"ScoSuperAdmin";"POST";"{ ""password"": ""too_simple"" }"
+"user-role-add";"/user/10/role/Observateur/add";"ScoSuperAdmin";"POST";
+"user-role-remove";"/user/10/role/Observateur/remove";"ScoSuperAdmin";"POST";
+"role-create";"/role/create/customRole";"ScoSuperAdmin";"POST";"{""permissions"": [""ScoView"", ""ScoUsersView""]}"
+"role-remove_permission";"/role/customRole/remove_permission/ScoUsersView";"ScoSuperAdmin";"POST";
+"role-add_permission";"/role/customRole/add_permission/ScoUsersView";"ScoSuperAdmin";"POST";
+"role-edit";"/role/customRole/edit";"ScoSuperAdmin";"POST";"{ ""name"" : ""LaveurDeVitres"", ""permissions"" : [ ""ScoView"" ] }"
+"role-delete";"/role/customRole/delete";"ScoSuperAdmin";"POST";
+"logos";"/logos";"ScoSuperAdmin";"GET";
+"logo";"/logo/B";"ScoSuperAdmin";"GET";
+"departement-logos";"/departement/TAPI/logos";"ScoSuperAdmin";"GET";
+"departement-logos";"/departement/id/1/logos";"ScoSuperAdmin";"GET";
+"departement-logo";"/departement/TAPI/logo/D";"ScoSuperAdmin";"GET";
+"departement-logo";"/departement/id/1/logo/D";"ScoSuperAdmin";"GET";
+"test-pdf";"/etudiant/nip/11/formsemestre/1/bulletin/pdf";"ScoView";"GET";
+"test-pdf";"/etudiant/nip/11/formsemestre/1/bulletin/pdf";"ScoView";"GET";
+"test-pdf";"/etudiant/etudid/11/formsemestre/1/bulletin/short/pdf";"ScoView";"GET";
+"test-pdf";"/etudiant/ine/INE11/formsemestre/1/bulletin/short/pdf";"ScoView";"GET";
+"test-pdf";"/etudiant/nip/11/formsemestre/1/bulletin/short/pdf";"ScoView";"GET";
+"test-pdf";"/etudiant/etudid/11/formsemestre/1/bulletin/pdf";"ScoView";"GET";
+"test-pdf";"/etudiant/etudid/11/formsemestre/1/bulletin/short";"ScoView";"GET";
+"test-pdf";"/etudiant/ine/INE11/formsemestre/1/bulletin/short";"ScoView";"GET";
+"test-pdf";"/etudiant/nip/11/formsemestre/1/bulletin/short";"ScoView";"GET";
+"test-pdf";"/etudiant/etudid/11/formsemestre/1/bulletin";"ScoView";"GET";
+"test-pdf";"/etudiant/ine/INE11/formsemestre/1/bulletin";"ScoView";"GET";
+"test-pdf";"/etudiant/nip/11/formsemestre/1/bulletin";"ScoView";"GET";