From 66ef5faecb9b7c7673143a23c1e6d91884899409 Mon Sep 17 00:00:00 2001 From: Jean-Marie PLACE Date: Sat, 12 Nov 2022 15:35:36 +0100 Subject: [PATCH] improve_csv_read --- tests/api/make_samples.py | 40 ++++++++++----- tests/api/samples.csv | 99 ------------------------------------ tests/ressources/samples.csv | 99 ++++++++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 112 deletions(-) delete mode 100644 tests/api/samples.csv create mode 100644 tests/ressources/samples.csv diff --git a/tests/api/make_samples.py b/tests/api/make_samples.py index 576f1263..e262052c 100644 --- a/tests/api/make_samples.py +++ b/tests/api/make_samples.py @@ -40,16 +40,19 @@ Le tout mis en forme au format markdown et rangé dans le répertoire DATA_DIR ( TODO: ajouter un argument au script permettant de ne générer qu'un seul fichier (exemple: `python make_samples.py nom_exemple`) """ +import numpy as np import os import shutil import sys import re from collections import defaultdict from pprint import pprint as pp -from pprint import pformat as pf import urllib3 import json + +from pandas import read_csv + from setup_test_api import ( API_PASSWORD, API_URL, @@ -63,6 +66,7 @@ from setup_test_api import ( ) DATA_DIR = "/tmp/samples/" +SAMPLES_FILENAME = "tests/ressources/samples.csv" class Sample: @@ -143,7 +147,12 @@ class Samples: self.entries = defaultdict(lambda: set()) self.entry_names = entry_names - def add_sample(self, entry, url, method="GET", permission="ScoView", content=None): + def add_sample(self, line): + entry = line["entry_name"] + url = line["url"] + method = line["method"] + permission = line["permission"] + content = line["content"] if self.entry_names is None or entry in self.entry_names: if method[0] == "#": detail = "**ignored**" @@ -185,19 +194,24 @@ def make_samples(): shutil.rmtree(DATA_DIR) os.mkdir(DATA_DIR) else: - os.mkdir("/tmp/samples") + os.mkdir(DATA_DIR) samples = Samples(entry_names) - samples_file = os.path.dirname(__file__) + "/samples.csv" - with open(samples_file) as f: - L = [x[:-1].split("\t") for x in f] - for line in L[1:]: - entry_name = line[0] - url = line[1] - permission = line[2] if line[2] != "" else "ScoView" - method = line[3] if line[3] != "" else "GET" - content = line[4] if len(line) > 4 else "" - samples.add_sample(entry_name, url, method, permission, content) + df = read_csv( + SAMPLES_FILENAME, + sep=";", + quotechar='"', + dtype={ + "entry_name": str, + "url": str, + "permission": str, + "method": str, + "content": str, + }, + keep_default_na=False, + ) + df = df.reset_index() + df.apply(lambda line: samples.add_sample(line), axis=1) samples.dump() return samples diff --git a/tests/api/samples.csv b/tests/api/samples.csv deleted file mode 100644 index e12b8f9a..00000000 --- a/tests/api/samples.csv +++ /dev/null @@ -1,99 +0,0 @@ -reference url permission method content -departements /departements GET -departements-ids /departements_ids GET -departement /departement/TAPI GET -departement /departement/id/1 GET -departement-etudiants /departement/TAPI/etudiants GET -departement-etudiants /departement/id/1/etudiants GET -departement-formsemestres_ids /departement/TAPI/formsemestres_ids GET -departement-formsemestres_ids /departement/id/1/formsemestres_ids GET -departement-formsemestres-courants /departement/TAPI/formsemestres_courants GET -departement-formsemestres-courants /departement/id/1/formsemestres_courants GET -departement-create /departement/create ScoSuperAdmin POST {"acronym": "NEWONE" , "visible": true} -departement-edit /departement/NEWONE/edit ScoSuperAdmin POST {"visible": false} -departement-delete /departement/NEWONE/delete ScoSuperAdmin POST -etudiants-courants /etudiants/courants?date_courante=2022-07-20 ScoView GET -etudiants-courants /etudiants/courants/long?date_courante=2022-07-20 ScoView GET -etudiant /etudiant/etudid/11 ScoView GET -etudiant /etudiant/nip/11 ScoView GET -etudiant /etudiant/ine/INE11 ScoView GET -etudiants-clef /etudiants/etudid/11 ScoView GET -etudiants-clef /etudiants/ine/INE11 ScoView GET -etudiants-clef /etudiants/nip/11 ScoView GET -etudiant-formsemestres /etudiant/etudid/11/formsemestres ScoView GET -etudiant-formsemestres /etudiant/ine/INE11/formsemestres ScoView GET -etudiant_formsemestres /etudiant/nip/11/formsemestres ScoView GET -etudiant-formsemestre-bulletin /etudiant/etudid/11/formsemestre/1/bulletin ScoView GET -etudiant-formsemestre-bulletin /etudiant/ine/INE11/formsemestre/1/bulletin ScoView GET -etudiant-formsemestre-bulletin /etudiant/nip/11/formsemestre/1/bulletin ScoView GET -etudiant-formsemestre-bulletin /etudiant/nip/11/formsemestre/1/bulletin/short/pdf ScoView GET -etudiant-formsemestre-groups /etudiant/etudid/11/formsemestre/1/groups ScoView GET -formations /formations ScoView GET -formations_ids /formations_ids ScoView GET -formation /formation/1 ScoView GET -formation-export /formation/1/export ScoView GET -formation-export /formation/1/export_with_ids ScoView GET -formation-referentiel_competences /formation/1/referentiel_competences ScoView GET -moduleimpl /moduleimpl/1 ScoView GET -formsemestre /formsemestre/1 ScoView GET -formsemestres-query /formsemestres/query?annee_scolaire=2022&etape_apo=A2 ScoView GET -formsemestres-query /formsemestres/query?nip=11 ScoView GET -formsemestre-bulletins /formsemestre/1/bulletins ScoView GET -formsemestre-programme /formsemestre/1/programme ScoView GET -formsemestre-etudiants /formsemestre/1/etudiants ScoView GET -formsemestre-etudiants /formsemestre/1/etudiants/long ScoView GET -formsemestre-etudiants-query /formsemestre/1/etudiants/query?etat=D ScoView GET -formsemestre-etat_evals /formsemestre/1/etat_evals ScoView GET -formsemestre-resultats /formsemestre/1/resultats ScoView GET -formsemestre-decisions_jury /formsemestre/1/decisions_jury ScoView GET -formsemestre-partitions /formsemestre/1/partitions ScoView GET -partition /partition/1 ScoView GET -group-etudiants /group/1/etudiants ScoView GET -group-etudiants-query /group/1/etudiants/query?etat=D ScoView GET -moduleimpl-evaluations /moduleimpl/1/evaluations ScoView GET -evaluation-notes /evaluation/1/notes ScoView GET -user /user/1 ScoView GET -users-query /users/query?starts_with=u_ ScoView GET -permissions /permissions ScoView GET -roles /roles ScoView GET -role /role/Observateur ScoView GET -group-set_etudiant /group/1/set_etudiant/10 ScoSuperAdmin POST -group-remove_etudiant /group/1/remove_etudiant/10 ScoSuperAdmin POST -partition-group-create /partition/1/group/create ScoSuperAdmin POST {"group_name": "NEW_GROUP"} -group-edit /group/2/edit ScoSuperAdmin POST {"group_name": "NEW_GROUP2"} -group-delete /group/2/delete ScoSuperAdmin POST -formsemestre-partition-create /formsemestre/1/partition/create ScoSuperAdmin POST {"partition_name": "PART"} -formsemestre-partitions-order /formsemestre/1/partitions/order ScoSuperAdmin POST [ 1 ] -partition-edit /partition/1/edit ScoSuperAdmin POST {"partition_name":"P2BIS", "numero":3,"bul_show_rank":true,"show_in_lists":false, "groups_editable":true} -partition-remove_etudiant /partition/2/remove_etudiant/10 ScoSuperAdmin POST -partition-groups-order /partition/1/groups/order ScoSuperAdmin POST [ 1 ] -partition-delete /partition/2/delete ScoSuperAdmin POST -user-create /user/create ScoSuperAdmin POST {"user_name": "alain", "dept": null, "nom": "alain", "prenom": "bruno", "active": true } -user-edit /user/10/edit ScoSuperAdmin POST { "dept": "TAPI", "nom": "alain2", "prenom": "bruno2", "active": false } -user-password /user/3/password ScoSuperAdmin POST { "password": "rePlaCemeNT456averylongandcomplicated" } -user-password /user/3/password ScoSuperAdmin POST { "password": "too_simple" } -user-role-add /user/10/role/Observateur/add ScoSuperAdmin POST -user-role-remove /user/10/role/Observateur/remove ScoSuperAdmin POST -role-create /role/create/customRole ScoSuperAdmin POST {"permissions": ["ScoView", "ScoUsersView"]} -role-remove_permission /role/customRole/remove_permission/ScoUsersView ScoSuperAdmin POST -role-add_permission /role/customRole/add_permission/ScoUsersView ScoSuperAdmin POST -role-edit /role/customRole/edit ScoSuperAdmin POST { "name" : "LaveurDeVitres", "permissions" : [ "ScoView" ] } -role-delete /role/customRole/delete ScoSuperAdmin POST -logos /logos ScoSuperAdmin GET -logo /logo/demo ScoSuperAdmin GET -departement-logos /departement/TAPI/logos ScoSuperAdmin GET -departement-logos /departement/id/1/logos ScoSuperAdmin GET -departement-logo /departement/TAPI/logo/demo ScoSuperAdmin GET -departement-logo /departement/id/1/logo/demo ScoSuperAdmin GET -test-pdf /etudiant/nip/11/formsemestre/1/bulletin/pdf ScoView GET -test-pdf /etudiant/nip/11/formsemestre/1/bulletin/pdf ScoView GET -test-pdf /etudiant/etudid/11/formsemestre/1/bulletin/short/pdf ScoView GET -test-pdf /etudiant/ine/INE11/formsemestre/1/bulletin/short/pdf ScoView GET -test-pdf /etudiant/nip/11/formsemestre/1/bulletin/short/pdf ScoView GET -test-pdf /etudiant/etudid/11/formsemestre/1/bulletin/pdf ScoView GET -test-pdf /etudiant/etudid/11/formsemestre/1/bulletin/short ScoView GET -test-pdf /etudiant/ine/INE11/formsemestre/1/bulletin/short ScoView GET -test-pdf /etudiant/nip/11/formsemestre/1/bulletin/short ScoView GET -test-pdf /etudiant/etudid/11/formsemestre/1/bulletin ScoView GET -test-pdf /etudiant/ine/INE11/formsemestre/1/bulletin ScoView GET -test-pdf /etudiant/nip/11/formsemestre/1/bulletin ScoView GET diff --git a/tests/ressources/samples.csv b/tests/ressources/samples.csv new file mode 100644 index 00000000..819d39c2 --- /dev/null +++ b/tests/ressources/samples.csv @@ -0,0 +1,99 @@ +"entry_name";"url";"permission";"method";"content" +"departements";"/departements";"ScoView";"GET"; +"departements-ids";"/departements_ids";"ScoView";"GET"; +"departement";"/departement/TAPI";"ScoView";"GET"; +"departement";"/departement/id/1";"ScoView";"GET"; +"departement-etudiants";"/departement/TAPI/etudiants";"ScoView";"GET"; +"departement-etudiants";"/departement/id/1/etudiants";"ScoView";"GET"; +"departement-formsemestres_ids";"/departement/TAPI/formsemestres_ids";"ScoView";"GET"; +"departement-formsemestres_ids";"/departement/id/1/formsemestres_ids";"ScoView";"GET"; +"departement-formsemestres-courants";"/departement/TAPI/formsemestres_courants";"ScoView";"GET"; +"departement-formsemestres-courants";"/departement/id/1/formsemestres_courants";"ScoView";"GET"; +"departement-create";"/departement/create";"ScoSuperAdmin";"POST";"{""acronym"": ""NEWONE"" , ""visible"": true}" +"departement-edit";"/departement/NEWONE/edit";"ScoSuperAdmin";"POST";"{""visible"": false}" +"departement-delete";"/departement/NEWONE/delete";"ScoSuperAdmin";"POST"; +"etudiants-courants";"/etudiants/courants?date_courante=2022-07-20";"ScoView";"GET"; +"etudiants-courants";"/etudiants/courants/long?date_courante=2022-07-20";"ScoView";"GET"; +"etudiant";"/etudiant/etudid/11";"ScoView";"GET"; +"etudiant";"/etudiant/nip/11";"ScoView";"GET"; +"etudiant";"/etudiant/ine/INE11";"ScoView";"GET"; +"etudiants-clef";"/etudiants/etudid/11";"ScoView";"GET"; +"etudiants-clef";"/etudiants/ine/INE11";"ScoView";"GET"; +"etudiants-clef";"/etudiants/nip/11";"ScoView";"GET"; +"etudiant-formsemestres";"/etudiant/etudid/11/formsemestres";"ScoView";"GET"; +"etudiant-formsemestres";"/etudiant/ine/INE11/formsemestres";"ScoView";"GET"; +"etudiant_formsemestres";"/etudiant/nip/11/formsemestres";"ScoView";"GET"; +"etudiant-formsemestre-bulletin";"/etudiant/etudid/11/formsemestre/1/bulletin";"ScoView";"GET"; +"etudiant-formsemestre-bulletin";"/etudiant/ine/INE11/formsemestre/1/bulletin";"ScoView";"GET"; +"etudiant-formsemestre-bulletin";"/etudiant/nip/11/formsemestre/1/bulletin";"ScoView";"GET"; +"etudiant-formsemestre-bulletin";"/etudiant/nip/11/formsemestre/1/bulletin/short/pdf";"ScoView";"GET"; +"etudiant-formsemestre-groups";"/etudiant/etudid/11/formsemestre/1/groups";"ScoView";"GET"; +"formations";"/formations";"ScoView";"GET"; +"formations_ids";"/formations_ids";"ScoView";"GET"; +"formation";"/formation/1";"ScoView";"GET"; +"formation-export";"/formation/1/export";"ScoView";"GET"; +"formation-export";"/formation/1/export_with_ids";"ScoView";"GET"; +"formation-referentiel_competences";"/formation/1/referentiel_competences";"ScoView";"GET"; +"moduleimpl";"/moduleimpl/1";"ScoView";"GET"; +"formsemestre";"/formsemestre/1";"ScoView";"GET"; +"formsemestres-query";"/formsemestres/query?annee_scolaire=2022&etape_apo=A2";"ScoView";"GET"; +"formsemestres-query";"/formsemestres/query?nip=11";"ScoView";"GET"; +"formsemestre-bulletins";"/formsemestre/1/bulletins";"ScoView";"GET"; +"formsemestre-programme";"/formsemestre/1/programme";"ScoView";"GET"; +"formsemestre-etudiants";"/formsemestre/1/etudiants";"ScoView";"GET"; +"formsemestre-etudiants";"/formsemestre/1/etudiants/long";"ScoView";"GET"; +"formsemestre-etudiants-query";"/formsemestre/1/etudiants/query?etat=D";"ScoView";"GET"; +"formsemestre-etat_evals";"/formsemestre/1/etat_evals";"ScoView";"GET"; +"formsemestre-resultats";"/formsemestre/1/resultats";"ScoView";"GET"; +"formsemestre-decisions_jury";"/formsemestre/1/decisions_jury";"ScoView";"GET"; +"formsemestre-partitions";"/formsemestre/1/partitions";"ScoView";"GET"; +"partition";"/partition/1";"ScoView";"GET"; +"group-etudiants";"/group/1/etudiants";"ScoView";"GET"; +"group-etudiants-query";"/group/1/etudiants/query?etat=D";"ScoView";"GET"; +"moduleimpl-evaluations";"/moduleimpl/1/evaluations";"ScoView";"GET"; +"evaluation-notes";"/evaluation/1/notes";"ScoView";"GET"; +"user";"/user/1";"ScoView";"GET"; +"users-query";"/users/query?starts_with=u_";"ScoView";"GET"; +"permissions";"/permissions";"ScoView";"GET"; +"roles";"/roles";"ScoView";"GET"; +"role";"/role/Observateur";"ScoView";"GET"; +"group-set_etudiant";"/group/1/set_etudiant/10";"ScoSuperAdmin";"POST"; +"group-remove_etudiant";"/group/1/remove_etudiant/10";"ScoSuperAdmin";"POST"; +"partition-group-create";"/partition/1/group/create";"ScoSuperAdmin";"POST";"{""group_name"": ""NEW_GROUP""}" +"group-edit";"/group/2/edit";"ScoSuperAdmin";"POST";"{""group_name"": ""NEW_GROUP2""}" +"group-delete";"/group/2/delete";"ScoSuperAdmin";"POST"; +"formsemestre-partition-create";"/formsemestre/1/partition/create";"ScoSuperAdmin";"POST";"{""partition_name"": ""PART""} " +"formsemestre-partitions-order";"/formsemestre/1/partitions/order";"ScoSuperAdmin";"POST";"[ 1 ]" +"partition-edit";"/partition/1/edit";"ScoSuperAdmin";"POST";"{""partition_name"":""P2BIS"", ""numero"":3,""bul_show_rank"":true,""show_in_lists"":false, ""groups_editable"":true}" +"partition-remove_etudiant";"/partition/2/remove_etudiant/10";"ScoSuperAdmin";"POST"; +"partition-groups-order";"/partition/1/groups/order";"ScoSuperAdmin";"POST";"[ 1 ]" +"partition-delete";"/partition/2/delete";"ScoSuperAdmin";"POST"; +"user-create";"/user/create";"ScoSuperAdmin";"POST";"{""user_name"": ""alain"", ""dept"": null, ""nom"": ""alain"", ""prenom"": ""bruno"", ""active"": true }" +"user-edit";"/user/10/edit";"ScoSuperAdmin";"POST";"{ ""dept"": ""TAPI"", ""nom"": ""alain2"", ""prenom"": ""bruno2"", ""active"": false }" +"user-password";"/user/3/password";"ScoSuperAdmin";"POST";"{ ""password"": ""rePlaCemeNT456averylongandcomplicated"" }" +"user-password";"/user/3/password";"ScoSuperAdmin";"POST";"{ ""password"": ""too_simple"" }" +"user-role-add";"/user/10/role/Observateur/add";"ScoSuperAdmin";"POST"; +"user-role-remove";"/user/10/role/Observateur/remove";"ScoSuperAdmin";"POST"; +"role-create";"/role/create/customRole";"ScoSuperAdmin";"POST";"{""permissions"": [""ScoView"", ""ScoUsersView""]}" +"role-remove_permission";"/role/customRole/remove_permission/ScoUsersView";"ScoSuperAdmin";"POST"; +"role-add_permission";"/role/customRole/add_permission/ScoUsersView";"ScoSuperAdmin";"POST"; +"role-edit";"/role/customRole/edit";"ScoSuperAdmin";"POST";"{ ""name"" : ""LaveurDeVitres"", ""permissions"" : [ ""ScoView"" ] }" +"role-delete";"/role/customRole/delete";"ScoSuperAdmin";"POST"; +"logos";"/logos";"ScoSuperAdmin";"GET"; +"logo";"/logo/B";"ScoSuperAdmin";"GET"; +"departement-logos";"/departement/TAPI/logos";"ScoSuperAdmin";"GET"; +"departement-logos";"/departement/id/1/logos";"ScoSuperAdmin";"GET"; +"departement-logo";"/departement/TAPI/logo/D";"ScoSuperAdmin";"GET"; +"departement-logo";"/departement/id/1/logo/D";"ScoSuperAdmin";"GET"; +"test-pdf";"/etudiant/nip/11/formsemestre/1/bulletin/pdf";"ScoView";"GET"; +"test-pdf";"/etudiant/nip/11/formsemestre/1/bulletin/pdf";"ScoView";"GET"; +"test-pdf";"/etudiant/etudid/11/formsemestre/1/bulletin/short/pdf";"ScoView";"GET"; +"test-pdf";"/etudiant/ine/INE11/formsemestre/1/bulletin/short/pdf";"ScoView";"GET"; +"test-pdf";"/etudiant/nip/11/formsemestre/1/bulletin/short/pdf";"ScoView";"GET"; +"test-pdf";"/etudiant/etudid/11/formsemestre/1/bulletin/pdf";"ScoView";"GET"; +"test-pdf";"/etudiant/etudid/11/formsemestre/1/bulletin/short";"ScoView";"GET"; +"test-pdf";"/etudiant/ine/INE11/formsemestre/1/bulletin/short";"ScoView";"GET"; +"test-pdf";"/etudiant/nip/11/formsemestre/1/bulletin/short";"ScoView";"GET"; +"test-pdf";"/etudiant/etudid/11/formsemestre/1/bulletin";"ScoView";"GET"; +"test-pdf";"/etudiant/ine/INE11/formsemestre/1/bulletin";"ScoView";"GET"; +"test-pdf";"/etudiant/nip/11/formsemestre/1/bulletin";"ScoView";"GET";