diff --git a/.gitignore b/.gitignore index 0aa5c7d..1d7434b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ /.env /venv -/cache.json +/*.json /*.csv +/*.svg # ---> Python # Byte-compiled / optimized / DLL files diff --git a/README.md b/README.md index 963bfc7..dc647d4 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,21 @@ partirait dans une autre filière après. Le cas paraît beaucoup plus douteux q ## FICHIERS +### get.py + +Le programme principal. Il prend en argument des acronymes de département (par exemple GEA ou INFO) et fabrique un graphe comportant les formations BUT de ce département (ou ces départements dans le même graphe, s'ils sont plusieurs sur la ligne de commande). + +Il faut un environnement virtuel pour que soient accessibles les bibliothèques Python pycairo, drawsvg, requests. A priori libcairo est optionnel, mais le graphe marchera moins bien sans. La bibliothèque (système) `libcairo2` doit aussi être installée (`apt install libcairo-dev` ou équivalent). + +On peut rajouter l'option `--techno` pour n'avoir que les bacs technos. + +### .env + +Fichier très important puisqu'il contient toutes les informations de connexion à la base Scodoc. +C'est un fichier CSV (séparateur virgule `,`) avec 4 lignes: username, password, server, baseyear. + +baseyear est l'année de départ de la cohorte étudiée. + ### redirect.csv Certains élèves ne reçoivent jamais de décision de jury lorsqu'ils quittent la cohorte, tout en n'étant pas démissionnaires. Ce sont des erreurs administratives, mais il est possible d'indiquer un *résultat de jury* fictif pour ces élèves. La plupart du temps, ce sont des élèves qui abandonnent la formation, et il suffit de leur donner le résultat NAR ou DEM. Dans d'autres cas, ça peut être des élèves en attente de décision parce que le jury n'a pas encore eu lieu, mais on sait déjà quel sera l'issue du jury (par exemple des notes élevés et un stage qui se déroule bien, ou au contraire pas de stage trouvé au mois de septembre). @@ -67,3 +82,10 @@ Possibilité de choisir les couleurs pour chacune des catégories. TRANSPARENT,#FFFFFF.0 RED,#000000 +### .json + +### .svg + +### best-.json + +Ce fichier contient le résultat d'une recherche heuristique pour avoir un graphe meilleur. Il peut être supprimé si le graphe ne s'améliore pas sur des lancements successifs. Il peut aussi être modifié à la main. \ No newline at end of file diff --git a/get.py b/get.py index 4a42384..4b2f3e4 100755 --- a/get.py +++ b/get.py @@ -1,10 +1,12 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import requests from requests.auth import HTTPBasicAuth import csv, os, sys import json from datetime import datetime import re +import drawsvg + debug = True # Not used techno = False # global flag @@ -52,23 +54,40 @@ def read_conf(key): return {} +def write_conf(key, obj): + with open(f"{key}.json", "w") as f: + return json.dump(obj, f) + return {} + + conf = read_conf(orderkey) # Manage default values def conf_value(xkey: str): + defaults = { + "spacing": 14, + "thickness": 6, + "fontsize_name": 10, + "fontsize_count": 14, + "width": 1300, + "height": 900, + "hmargin": 20, + "parcours_separator": "/", + "year_separator": "", + "rank_separator": "", + "diplome_separator": "", + } if xkey in conf: return conf[xkey] - if xkey == "width": - return 1300 - if xkey == "height": - return 900 + if xkey in defaults: + return defaults[xkey] if xkey[-9:] == "separator": return " " if xkey == "nick": - return "{diplome}{rank}{department}{modalite}{parcours}" + return "{diplome}{rank}{multidepartment}{modalite}{parcours}" if xkey == "extnick": - return "{rank}{diplomenobut}{modaliteshort}" + return "{rank}{multidepartment}{diplomenobut}{modaliteshort}" if xkey == "orders": return [[], [], [], [], []] return {} @@ -78,21 +97,38 @@ def conf_value(xkey: str): # This file should be kept really safe # Currently, only baseyear = 2021 has been tested -# TODO: a CSV file would be more declarative and easier to manage -from dotenv import dotenv_values +server, username, password, baseyear = "", "", "", 0 -def read_secrets(): - global server, username, password, baseyear - config = dotenv_values(".env") - server = config["server"] - username = config["username"] - password = config["password"] - baseyear = int(config["baseyear"]) +def read_secrets(filename): + keys = ["server", "username", "password", "baseyear"] + integers = ["baseyear"] + if os.path.exists(filename): + with open(filename, newline="") as csvfile: + csvreader = csv.reader(csvfile, delimiter=",", quotechar='"') + found = 0 + for row in csvreader: + for p, k in enumerate(keys): + if row[0] == k: + if k in integers: + globals()[k] = int(row[1]) + else: + globals()[k] = row[1] + found |= 0x1 << p + if found != 0xF: + print(f'Des paramètres manquent dans "{filename}" ({found}).') + for p, k in enumerate(keys): + if found & (0x1 << p) == 0: + if k in globals(): + g = '"' + globals()[k] + '"' + else: + g = None + print(f"{k} = {g}") + sys.exit(1) -read_secrets() +read_secrets(".env") student = {} @@ -327,6 +363,12 @@ def analyse_student(semobj, etud, year=None): ) else: nick = nick.replace("{department}", "") + if len(department) > 0 and len(depts) > 1: + nick = nick.replace( + "{multidepartment}", conf_value("department_separator") + department + ) + else: + nick = nick.replace("{multidepartment}", "") if len(diplome) > 0: nick = nick.replace("{diplome}", conf_value("diplome_separator") + diplome) else: @@ -344,6 +386,9 @@ def analyse_student(semobj, etud, year=None): else: nick = nick.replace("{parcours}", "") formsem_department[str(semobj["id"])] = department + if nick == " BUT 1 GEA EXT": + print(department, diplome, rank, modalite, parcours, nick) + sys.exit(0) return department, diplome, rank, modalite, parcours, nick @@ -367,86 +412,88 @@ cohort_nip = set() def analyse_depts(): for dept in depts: formsems = get_formsem_from_dept(dept) - for semid in formsems: - # Check if this is a part of the cohort - # or a future/old semester - sem = cache["sem"][str(semid)] - if sem["semestre_id"] < 0: - year = 1 - else: - year = (sem["semestre_id"] + 1) // 2 - offset = sem["annee_scolaire"] - baseyear - year + 1 - if offset < 0 and offset > -4: - oldsems.add(str(semid)) - oldsemsdept[semid] = dept - if offset > 0 and offset < 4: - futuresems.add(str(semid)) - futuresemsdept[semid] = dept - if offset != 0: - continue - if offset == 0 and sem["formation"]["type_parcours"] != 700: - continue - # This is a BUT semester, part of the cohort - # 0,1 : preceding year ; 2-7 : cohort ; 8+ : future - if sem["semestre_id"] < 0: - bucket = 1 - else: - bucket = str(int(sem["semestre_id"] - 1)) - # Ici, le semestre est donc un semestre intéressant - # On prélève tous les étudiants, et on remplit leur cursus - etuds = get_etuds_from_formsem(dept, semid) - jurys = get_jury_from_formsem(dept, semid) - key = sem["titre_num"] - for etud in etuds: - etudid = etud["id"] - if etudid in student: - studentsummary = student[etudid] + for semid in formsems: + # Check if this is a part of the cohort + # or a future/old semester + sem = cache["sem"][str(semid)] + if sem["semestre_id"] < 0: + year = 1 else: - studentsummary = {} - studentsummary["cursus"] = {} # Cursus is semid - studentsummary["etudid"] = {} # useful when merging students - studentsummary["pseudodept"] = {} # pseudo-dept for interdept - studentsummary["diplome"] = {} # diplome name - studentsummary["rank"] = {} # rank - studentsummary["modalite"] = {} # modalite - studentsummary["parcours"] = {} # parcours - studentsummary["nickname"] = {} # nick - studentsummary["dept"] = dept # useful when merging students - studentsummary["bac"] = "" # usually - department, diplome, rank, modalite, parcours, nick = analyse_student( - sem, etud, year - ) - if "bac" in etud["admission"]: - studentsummary["bac"] = etud["admission"]["bac"] - else: - studentsummary["bac"] = "INCONNU" - bacs.add(studentsummary["bac"]) - # We skip non-techno students if we are in techno mode - # If we want a mixed reporting, maybe we should change this - if techno and studentsummary["bac"][:2] != "ST": # TODO: change this + year = (sem["semestre_id"] + 1) // 2 + offset = sem["annee_scolaire"] - baseyear - year + 1 + if offset < 0 and offset > -4: + oldsems.add(str(semid)) + oldsemsdept[semid] = dept + if offset > 0 and offset < 4: + futuresems.add(str(semid)) + futuresemsdept[semid] = dept + if offset != 0: continue - if bucket in studentsummary["cursus"]: - semestreerreur = int(bucket) + 1 - print( - f"// Élève {etudid} dans deux semestres à la fois : S{semestreerreur}, semestres {studentsummary['cursus'][bucket]} et {semid}" + if sem["formation"]["type_parcours"] != 700: + continue + if sem["modalite"] == "EXT": + continue + # This is a BUT semester, part of the cohort + # 0,1 : preceding year ; 2-7 : cohort ; 8+ : future + if sem["semestre_id"] < 0: + bucket = 1 + else: + bucket = str(int(sem["semestre_id"] - 1)) + # Ici, le semestre est donc un semestre intéressant + # On prélève tous les étudiants, et on remplit leur cursus + etuds = get_etuds_from_formsem(dept, semid) + jurys = get_jury_from_formsem(dept, semid) + key = sem["titre_num"] + for etud in etuds: + etudid = etud["id"] + if etudid in student: + studentsummary = student[etudid] + else: + studentsummary = {} + studentsummary["cursus"] = {} # Cursus is semid + studentsummary["etudid"] = {} # useful when merging students + studentsummary["pseudodept"] = {} # pseudo-dept for interdept + studentsummary["diplome"] = {} # diplome name + studentsummary["rank"] = {} # rank + studentsummary["modalite"] = {} # modalite + studentsummary["parcours"] = {} # parcours + studentsummary["nickname"] = {} # nick + studentsummary["dept"] = dept # useful when merging students + studentsummary["bac"] = "" # usually + department, diplome, rank, modalite, parcours, nick = analyse_student( + sem, etud, year ) - if "dept" in studentsummary and studentsummary["dept"] != dept: - print( - f"// Élève ayant changé de département {dept},{studentsummary['dept']}" - ) - # department, diplome, rank, modalite, parcours, nick = analyse_student( - studentsummary["cursus"][bucket] = semid - studentsummary["etudid"][bucket] = etudid - studentsummary["pseudodept"][bucket] = department - studentsummary["diplome"][bucket] = diplome - studentsummary["rank"][bucket] = rank - studentsummary["modalite"][bucket] = modalite - studentsummary["parcours"][bucket] = parcours - studentsummary["nickname"][bucket] = nick - studentsummary["debug"] = etud["sort_key"] # TODO: REMOVE - studentsummary["unid"] = etud["code_nip"] - cohort_nip.add(etud["code_nip"]) - student[etudid] = studentsummary + if "bac" in etud["admission"]: + studentsummary["bac"] = etud["admission"]["bac"] + else: + studentsummary["bac"] = "INCONNU" + bacs.add(studentsummary["bac"]) + # We skip non-techno students if we are in techno mode + # If we want a mixed reporting, maybe we should change this + if techno and studentsummary["bac"][:2] != "ST": # TODO: change this + continue + if bucket in studentsummary["cursus"]: + semestreerreur = int(bucket) + 1 + print( + f"// Élève {etudid} dans deux semestres à la fois : S{semestreerreur}, semestres {studentsummary['cursus'][bucket]} et {semid}" + ) + if "dept" in studentsummary and studentsummary["dept"] != dept: + print( + f"// Élève ayant changé de département {dept},{studentsummary['dept']}" + ) + # department, diplome, rank, modalite, parcours, nick = analyse_student( + studentsummary["cursus"][bucket] = semid + studentsummary["etudid"][bucket] = etudid + studentsummary["pseudodept"][bucket] = department + studentsummary["diplome"][bucket] = diplome + studentsummary["rank"][bucket] = rank + studentsummary["modalite"][bucket] = modalite + studentsummary["parcours"][bucket] = parcours + studentsummary["nickname"][bucket] = nick + studentsummary["debug"] = etud["sort_key"] # TODO: REMOVE + studentsummary["unid"] = etud["code_nip"] + cohort_nip.add(etud["code_nip"]) + student[etudid] = studentsummary analyse_depts() @@ -506,10 +553,9 @@ def allseeingodin(): student[etudid]["oldsem"] = oldstudents[unid][0] if unid in futurestudents: student[etudid]["future"] = futurestudents[unid] - + for unid in duplicates: lastsem = -1 best = [] - for unid in duplicates: for suppidx in duplicates[unid][1:]: supp = student[suppidx] if str(lastsem) in supp["cursus"]: @@ -520,10 +566,11 @@ def allseeingodin(): best = [suppidx] break if len(best) > 1: - print( - f"// Warning: cannot chose last semester for NIP {unid}: " - + ", ".join(best) - ) + print(f"// Error: cannot chose last semester for NIP {unid}: ") + print(repr(best)) + for x in best: + print(cache["sem"][str(x)]) + sys.exit(6) bestid = best[0] base = student[bestid] for suppidx in duplicates[unid]: @@ -542,9 +589,10 @@ def allseeingodin(): "old", "oldsem", ): - for bucket in supp[skey]: - if bucket not in base[skey]: - base[skey][bucket] = supp[skey][bucket] + if skey in supp: + for bucket in supp[skey]: + if bucket not in base[skey]: + base[skey][bucket] = supp[skey][bucket] del student[suppidx] @@ -592,19 +640,19 @@ for etudid in student.keys(): nextbest = {} nextnickbest = {} for key in next: - max = 0 + imax = 0 best = None for key2 in next[key]: - if next[key][key2] > max: - max = next[key][key2] + if next[key][key2] > imax: + imax = next[key][key2] best = key2 nextbest[key] = best for key in nextnick: - max = 0 + imax = 0 best = None for key2 in nextnick[key]: - if nextnick[key][key2] > max: - max = nextnick[key][key2] + if nextnick[key][key2] > imax: + imax = nextnick[key][key2] best = key2 nextnickbest[key] = best @@ -664,8 +712,6 @@ for etudid in student.keys(): if ddd not in depts: depts.append(ddd) -dd = len(depts) - badred = {} goodred = {} failure = {} @@ -786,7 +832,10 @@ for etudid in student.keys(): firstyear - 1 - i ) else: - if etud["cursus"][str(firstyear * 2 - 2)] is not None: + if ( + str(firstyear * 2 - 2) in etud["cursus"] + and etud["cursus"][str(firstyear * 2 - 2)] is not None + ): startsem = str(firstyear * 2 - 2) else: startsem = str(firstyear * 2 - 1) @@ -802,6 +851,12 @@ for etudid in student.keys(): ) else: nick = nick.replace("{department}", "") + if len(department) > 0 and len(depts) > 1: + nick = nick.replace( + "{multidepartment}", conf_value("department_separator") + department + ) + else: + nick = nick.replace("{multidepartment}", "") if len(diplome) > 0: nick = nick.replace("{diplome}", conf_value("diplome_separator") + diplome) else: @@ -857,191 +912,556 @@ for etudid in student.keys(): bags[i][nstart][nend] += 1 -layers = [[], [], [], [], []] -finallayers = [[], [], [], [], []] -alllayers = [] -flatbags = [] -for i in range(4): - for u in bags[i]: - if u not in alllayers: - alllayers.append(u) - layers[i].append(u) - for v in bags[i][u]: - if v not in alllayers: - alllayers.append(v) - layers[i + 1].append(v) - flatbags.append([u, v, bags[i][u][v]]) -allowed = [] -nextallowed = [[], [], [], [], []] -weights = {} +# layers = [[], [], [], [], []] +# alllayers = [] +# flatbags = [] +# for i in range(4): +# for u in bags[i]: +# if u not in alllayers: +# alllayers.append(u) +# layers[i].append(u) +# for v in bags[i][u]: +# if v not in alllayers: +# alllayers.append(v) +# layers[i + 1].append(v) +# flatbags.append([u, v, bags[i][u][v]]) +# allowed = [] +# nextallowed = [[], [], [], [], []] +# weights = {} -orders = conf_value("orders") +# orders = conf_value("orders") -x = set(alllayers) -y = set() -for i in orders: - y = y.union(set(i)) +# x = set(alllayers) +# y = set() +# for i in orders: +# y = y.union(set(i)) -for i in range(5): - if len(orders[i]) > 0: - allowed.append(orders[i][0]) - for j in orders[i]: - if j in alllayers: - nextallowed[i].append(j) - for j, k in enumerate(orders[i]): - weights[k] = j + 1 - for u in layers[i]: - if u not in allowed and u not in nextallowed[i]: - allowed.append(u) -else: - for i in range(5): - allowed.extend(layers[i]) +# for i in range(5): +# if len(orders[i]) > 0: +# allowed.append(orders[i][0]) +# for j in orders[i]: +# if j in alllayers: +# nextallowed[i].append(j) +# for j, k in enumerate(orders[i]): +# weights[k] = j + 1 +# for u in layers[i]: +# if u not in allowed and u not in nextallowed[i]: +# allowed.append(u) +# else: +# for i in range(5): +# allowed.extend(layers[i]) -for bag in flatbags: +# for bag in flatbags: +# w = 0 +# if bag[0] in weights: +# w += weights[bag[0]] +# if bag[1] in weights: +# w += weights[bag[1]] +# bag.append(w) +# flatbags = sorted(flatbags, key=lambda x: x[-1]) + + +# orderedflatbags = [] +# finallayers = [[], [], [], [], []] + +# while len(flatbags) > 0: +# gotone = False +# for x in flatbags: +# if x[0] in allowed and x[1] in allowed: +# # print(f"{x} est pris") +# gotone = True +# orderedflatbags.append(x) +# flatbags.remove(x) +# # print(f"Choosing {x}") +# for i in range(5): +# if x[0] in layers[i] and x[0] not in finallayers[i]: +# finallayers[i].append(x[0]) +# if i < 4 and x[1] in layers[i + 1] and x[1] not in finallayers[i + 1]: +# finallayers[i + 1].append(x[1]) +# if x[0] in nextallowed[i]: +# # print(f"[{i}] Removing {x[0]} from {nextallowed[i]}") +# nextallowed[i].remove(x[0]) +# if x[1] in nextallowed[i]: +# # print(f"[{i}] Removing {x[1]} from {nextallowed[i]}") +# nextallowed[i].remove(x[1]) +# # print(f"[{i}] {nextallowed[i]}") +# if len(nextallowed[i]) > 0 and nextallowed[i][0] not in allowed: +# # print(f"[{i}] Allowing now {nextallowed[i][0]}") +# allowed.append(nextallowed[i][0]) +# break +# if not gotone: +# print("BUG") +# print(flatbags) +# print("---", allowed) +# print(nextallowed) +# sys.exit(3) + + +def node_color(x): + color = colors["NORMAL"] + if x[0:4] == "FAIL": + color = f"{colors['FAIL']} <<" + elif x[0:4] == "+DUT": + color = f"{colors['+DUT']} <<" + elif x[0:4] == "QUIT": + color = f"{colors['QUIT']} <<" + elif x[0:3] == "RED": + color = f"{colors['RED']} <<" + elif x[0:4] == "DIPL": + color = f"{colors['SUCCESS']} <<" + elif x[0:3] == "EXT": + color = f"{colors['NEW']} >>" + elif x[0:3] == "BUT": + color = f"{colors['NORMAL']}" + elif x[0:3] == "DUT": + color = f"{colors['OLD']} >>" + if x[-1] == "*": + color = f"{colors['TRANSPARENT']} >>" + return color + + +# def printout(): +# with open(f"sankeymatic_{orderkey}.txt", "w") as fout: + +# def output(*a, **b): +# b["file"] = fout +# print(*a, **b) + +# date_actuelle = datetime.now() +# date_formatee = date_actuelle.strftime("%m/%d/%Y %H:%M:%S") + +# output( +# f"// SankeyMATIC diagram inputs - Saved: {date_formatee}\n// https://sankeymatic.com/build/\n\n// === Nodes and Flows ===\n\n" +# ) + +# output("// THEME INFO") +# for c, cc in colors.items(): +# output(f"// !{c}:{cc}") +# output() + +# allnodes = [] +# for y in orderedflatbags: +# output(f"{y[0]} [{y[2]}] {y[1]}") +# allnodes.append(y[0]) +# allnodes.append(y[1]) +# allnodes = list(set(allnodes)) + +# nodes = {} +# for x in allnodes: +# color = node_color(x) +# if len(color): +# nodes[x] = color + +# for u in sorted(nodes.keys()): +# output(f":{u} {nodes[u]}") + +# height = conf_value("height") +# width = conf_value("width") +# output("\n\n// === Settings ===\n") +# output(f"size w {width}") +# output(f" h {height}") +# with open("trailer.txt", "r") as fichier: +# contenu = fichier.read() +# output(contenu) +# for ddd in depts: +# if entries[ddd] == 0: +# continue +# p1 = round(100 * diploma[ddd] / entries[ddd]) +# p2 = round(100 * (diploma[ddd] + reor2[ddd]) / entries[ddd]) +# p3 = round(100 * (failure[ddd] / entries[ddd])) +# p4 = round(100 * (failure[ddd] + badred[ddd] + reor1[ddd]) / entries[ddd]) + +# output(f"// Département {ddd}") +# output(f"// {entries[ddd]} Entrées") +# output(f"// {diploma[ddd]} Diplômes") +# output(f"// {reor2[ddd]} DUT") +# output(f"// {p1}-{p2}% de réussite") +# output(f"// {goodred[ddd]} Redoublements") +# output(f"// {reor1[ddd]} départs de la formation") +# output(f"// {badred[ddd]} redoublements autorisés non actés") +# output(f"// {failure[ddd]} échecs") +# output(f"// {p3}-{p4}% d'échecs") +# output(f"// {unknown[ddd]} inconnus") +# for x in strangecases: +# output(f"// {x}") + +# output(f'// orders["{orderkey}"] = {finallayers}') +# output(f"// bacs: {bacs}") + + +# printout() + + +def textwidth(text, font="Arial", fontsize=14): + try: + import cairo + except: + return len(text) * fontsize + surface = cairo.SVGSurface("undefined.svg", 1280, 200) + cr = cairo.Context(surface) + cr.select_font_face(font, cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_BOLD) + cr.set_font_size(fontsize) + xbearing, ybearing, width, height, xadvance, yadvance = cr.text_extents(text) + return width + + +def crossweight(node_position, node_layer, edges): w = 0 - if bag[0] in weights: - w += weights[bag[0]] - if bag[1] in weights: - w += weights[bag[1]] - bag.append(w) -flatbags = sorted(flatbags, key=lambda x: x[-1]) + for e in edges: + for ee in edges: + if node_layer[e[0]] != node_layer[ee[0]]: + continue + if node_layer[e[1]] != node_layer[ee[1]]: + continue + if (node_position[e[0]] - node_position[ee[0]]) * ( + node_position[e[1]] - node_position[ee[1]] + ) < 0: + w += e[2] * ee[2] + return w -orderedflatbags = [] - -while len(flatbags) > 0: - gotone = False - for x in flatbags: - if x[0] in allowed and x[1] in allowed: - # print(f"{x} est pris") - gotone = True - orderedflatbags.append(x) - flatbags.remove(x) - # print(f"Choosing {x}") - for i in range(5): - if x[0] in layers[i] and x[0] not in finallayers[i]: - finallayers[i].append(x[0]) - if i < 4 and x[1] in layers[i + 1] and x[1] not in finallayers[i + 1]: - finallayers[i + 1].append(x[1]) - if x[0] in nextallowed[i]: - # print(f"[{i}] Removing {x[0]} from {nextallowed[i]}") - nextallowed[i].remove(x[0]) - if x[1] in nextallowed[i]: - # print(f"[{i}] Removing {x[1]} from {nextallowed[i]}") - nextallowed[i].remove(x[1]) - # print(f"[{i}] {nextallowed[i]}") - if len(nextallowed[i]) > 0 and nextallowed[i][0] not in allowed: - # print(f"[{i}] Allowing now {nextallowed[i][0]}") - allowed.append(nextallowed[i][0]) - break - if not gotone: - print("BUG") - print(flatbags) - print("---", allowed) - print(nextallowed) - sys.exit(3) +import random -def printout(): - with open(f"sankeymatic_{orderkey}.txt", "w") as fout: +def genetic_optimize(node_position, node_layer, edges): + oldcandidates = [] + l_indices = list(range(5)) + lays = [] + randomness_l = [] + for index in range(5): + lays.append([x for x in node_layer.keys() if node_layer[x] == index]) + if len(lays[index]) > 1: + for i in lays[index]: + randomness_l.append(index) + w = crossweight(node_position, node_layer, edges) + for i in range(20): + oldcandidates.append([node_position.copy(), w]) + w = crossweight(node_position, node_layer, edges) + for i in range(10): + n = node_position.copy() + l_idx = random.choice(randomness_l) + q = lays[l_idx].copy() + k = 0 + while len(q) > 0: + nn = random.choice(q) + q.remove(nn) + n[nn] = k + k += 1 + oldcandidates.append([n, w]) + candidates = oldcandidates + for i in range(300): + oldcandidates = candidates + oldcandidates.sort(key=lambda x: x[1]) + candidates = oldcandidates[:30] + while len(candidates) < 60: + # mutate some random candidate + candidate = random.choice(candidates)[0] + new_position = candidate.copy() # Copier la position pour la muter + l_idx = random.choice(randomness_l) + swapa = random.choice(lays[l_idx]) + swapb = random.choice(lays[l_idx]) + while swapa == swapb: + swapb = random.choice(lays[l_idx]) + tmp = new_position[swapa] + new_position[swapa] = new_position[swapb] + new_position[swapb] = tmp + w = crossweight(new_position, node_layer, edges) + candidates.append([new_position, w]) + while len(candidates) < 90: + # mutate some random candidate + candidate = random.choice(candidates)[0] + new_position = candidate.copy() # Copier la position pour la muter + l_idx = random.choice(randomness_l) + startidx = random.randrange(len(lays[l_idx]) - 1) + stopidx = random.randrange(startidx + 1, len(lays[l_idx])) + for n in lays[l_idx]: + if new_position[n] >= startidx and new_position[n] < stopidx: + new_position[n] += 1 + elif new_position[n] == stopidx: + new_position[n] = startidx + w = crossweight(new_position, node_layer, edges) + candidates.append([new_position, w]) + while len(candidates) < 100: + # mutate some random candidate + candidate = random.choice(candidates)[0] + candidate2 = random.choice(candidates)[0] + new_position = candidate.copy() # Copier la position pour la muter + l_idx = random.choice(randomness_l) + for n in lays[l_idx]: + new_position[n] = candidate2[n] + w = crossweight(new_position, node_layer, edges) + candidates.append([new_position, w]) + candidates.sort(key=lambda x: x[1]) + orders = [] + best = candidates[0][0] + for i in range(5): + b = lays[i].copy() + b.sort(key=lambda x: best[x]) + orders.append(b) + print(orders) + print(candidates[0][1]) + return orders - def output(*a, **b): - b["file"] = fout - print(*a, **b) - date_actuelle = datetime.now() - date_formatee = date_actuelle.strftime("%m/%d/%Y %H:%M:%S") +def printsvg(): + padding = 4 + unit_ratio = 96 / 72 + thickness = conf_value("thickness") + fontsize_name = conf_value("fontsize_name") + fontsize_count = conf_value("fontsize_count") + spacing = conf_value("spacing") + height = conf_value("height") + hmargin = conf_value("hmargin") + width = conf_value("width") + node_structure = {} + layers = [[], [], [], [], []] + edges = [] + for layer, layernodes in enumerate(bags): + for startnode in layernodes: + if startnode[-1] == "*": + continue + for endnode in layernodes[startnode]: + if endnode[-1] == "*": + continue + weight = layernodes[startnode][endnode] + if endnode not in node_structure: + node_structure[endnode] = { + "prev": [[startnode, weight]], + "next": [], + "layer": layer + 1, + } + layers[layer + 1].append(endnode) + else: + node_structure[endnode]["prev"].append([startnode, weight]) + if startnode not in node_structure: + node_structure[startnode] = { + "prev": [], + "next": [[endnode, weight]], + "layer": layer, + } + layers[layer].append(startnode) + else: + node_structure[startnode]["next"].append([endnode, weight]) + edges.append([startnode, endnode, weight]) + node_position = {} + node_layer = {} + layer_structure = [ + {"olayer": []}, + {"olayer": []}, + {"olayer": []}, + {"olayer": []}, + {"olayer": []}, + ] - output( - f"// SankeyMATIC diagram inputs - Saved: {date_formatee}\n// https://sankeymatic.com/build/\n\n// === Nodes and Flows ===\n\n" + lastorders = read_conf("best-" + orderkey) + if lastorders != {}: + for i in range(5): + ls = layer_structure[i] + ord = lastorders[i] + for node in lastorders[i]: + if node in layers[i]: + ls["olayer"].append(node) + for node in layers[i]: + if node not in ls["olayer"]: + ls["olayer"].append(node) + for layer, layernodes in enumerate(layer_structure): + for j, n in enumerate(layernodes["olayer"]): + node_position[n] = j + node_layer[n] = layer + print(crossweight(node_position, node_layer, edges)) + else: + for layer, layernodes in enumerate(layers): + for j, n in enumerate(layernodes): + node_position[n] = j + node_layer[n] = layer + orders = genetic_optimize(node_position, node_layer, edges) + write_conf("best-" + orderkey, orders) + layer_structure = [ + {"olayer": []}, + {"olayer": []}, + {"olayer": []}, + {"olayer": []}, + {"olayer": []}, + ] + for i in range(5): + ls = layer_structure[i] + ord = orders[i] + for node in orders[i]: + if node in layers[i]: + ls["olayer"].append(node) + for node in layers[i]: + if node not in ls["olayer"]: + ls["olayer"].append(node) + for layer, layernodes in enumerate(layer_structure): + for j, n in enumerate(layernodes["olayer"]): + node_position[n] = j + node_layer[n] = layer + print(crossweight(node_position, node_layer, edges)) + density = [] + for i in range(5): + ls = layer_structure[i] + ls["num"] = len(ls["olayer"]) + ls["inout"] = 0 + for j in ls["olayer"]: + lhi = 0 + lho = 0 + k = node_structure[j] + for prev_node in k["prev"]: + lhi += prev_node[1] + for next_node in k["next"]: + lho += next_node[1] + k["size"] = max(lhi, lho) + k["in"] = lhi + k["out"] = lho + if lhi != lho and lhi * lho != 0: + print(f"BUG1: {j} {k} {lhi} {lho}") + ls["inout"] += k["size"] + ls["density"] = ls["inout"] / ( + spacing + height - spacing * ls["num"] - 2 * hmargin ) + density.append(ls["density"]) + realdensity = max(density) + columns = [] + l = 0 + for i in range(5): + l += width / 6 + columns.append(l) + for i in range(5): + ls = layer_structure[i] + supp_spacing = ( + spacing + + height + - 2 * hmargin + - spacing * ls["num"] + - ls["inout"] / realdensity + ) / (ls["num"] + 1) + cs = hmargin - spacing + for j in ls["olayer"]: + ns = node_structure[j] + ns["top"] = supp_spacing + spacing + cs + h = ns["size"] / realdensity + cs = ns["bottom"] = ns["top"] + h + d = drawsvg.Drawing(width, height, origin=(0, 0), id_prefix=orderkey) + g1 = drawsvg.Group() + g2 = drawsvg.Group() + g3 = drawsvg.Group() + g4 = drawsvg.Group() + font_offset = max(fontsize_count, fontsize_name) + for n in node_structure: + ns = node_structure[n] + col = node_color(n).split(" ")[0].split(".")[0] + ns["color"] = col + xpos = width / 6 * (ns["layer"] + 1) + r = drawsvg.Rectangle( + xpos - thickness, + ns["top"], + 2 * thickness, + ns["bottom"] - ns["top"], + fill=col, + stroke_width=0.2, + stroke="black", + ) + g1.append(r) + nw = textwidth(n, "Arial", fontsize_name) * unit_ratio + cw = textwidth(str(ns["size"]), "Arial", fontsize_count) * unit_ratio + gw = nw + cw + padding + ggw = gw + 2 * padding + nxpos = xpos - gw / 2 + cw + padding + nw / 2 + ypos = (ns["top"] + ns["bottom"]) / 2 + cxpos = cw / 2 - gw / 2 + xpos + rxpos = xpos + if ns["in"] == 0: + nxpos -= gw / 2 + padding + thickness + cxpos -= gw / 2 + padding + thickness + rxpos -= gw / 2 + padding + thickness + if ns["out"] == 0: + nxpos += gw / 2 + padding + thickness + cxpos += gw / 2 + padding + thickness + rxpos += gw / 2 + padding + thickness + t = drawsvg.Text( + n, + str(fontsize_name) + "pt", + nxpos, + ypos + fontsize_name / 2, + fill="black", + text_anchor="middle", + font_family="Arial", + ) + tt = drawsvg.Text( + str(ns["size"]), + str(fontsize_count) + "pt", + cxpos, + ypos + fontsize_count / 2, + fill="black", + text_anchor="middle", + font_family="Arial", + ) + g3.append(t) + g3.append(tt) + g4.append( + drawsvg.Rectangle( + rxpos - gw / 2 - padding, + ypos - font_offset / 2 - padding, + ggw, + font_offset + 2 * padding, + stroke="black", + stroke_width=0, + fill="white", + fill_opacity=".5", + rx=padding, + ry=padding, + ) + ) + for n in node_structure: + ns = node_structure[n] + ns["prev"].sort(key=lambda x: node_structure[x[0]]["top"]) + ns["next"].sort(key=lambda x: node_structure[x[0]]["top"]) + start = ns["top"] + for link in ns["prev"]: + ysize = link[-1] + link.append(start) + start += ysize / realdensity + link.append(start) + for n in node_structure: + ns = node_structure[n] + start = ns["top"] + for link in ns["next"]: + ysize = link[-1] + link.append(start) + start += ysize / realdensity + link.append(start) + targets = node_structure[link[0]] + target = None + for t in targets["prev"]: + if t[0] == n: + target = t + if target == None: + print(f"BUG: {n},{ns},{t}") + sys.exit(5) - output("// THEME INFO") - for c, cc in colors.items(): - output(f"// !{c}:{cc}") - output() - - allnodes = [] - for y in orderedflatbags: - output(f"{y[0]} [{y[2]}] {y[1]}") - allnodes.append(y[0]) - allnodes.append(y[1]) - allnodes = list(set(allnodes)) - - nodes = {} - for x in allnodes: - color = colors["NORMAL"] - if x[0:4] == "FAIL": - color = f"{colors['FAIL']} <<" - elif x[0:4] == "+DUT": - color = f"{colors['+DUT']} <<" - elif x[0:4] == "QUIT": - color = f"{colors['QUIT']} <<" - elif x[0:3] == "RED": - color = f"{colors['RED']} <<" - elif x[0:4] == "DIPL": - color = f"{colors['SUCCESS']} <<" - elif x[0:3] == "EXT": - color = f"{colors['NEW']} >>" - elif x[0:3] == "BUT": - color = f"{colors['NORMAL']}" - elif x[0:3] == "DUT": - color = f"{colors['OLD']} >>" - if x[-1] == "*": - color = f"{colors['TRANSPARENT']} >>" - if len(color): - nodes[x] = color - - for u in sorted(nodes.keys()): - output(f":{u} {nodes[u]}") - - height = conf_value("height") - width = conf_value("width") - output("\n\n// === Settings ===\n") - output(f"size w {width}") - output(f" h {height}") - with open("trailer.txt", "r") as fichier: - contenu = fichier.read() - output(contenu) - for ddd in depts: - p1 = round(100 * diploma[ddd] / entries[ddd]) - p2 = round(100 * (diploma[ddd] + reor2[ddd]) / entries[ddd]) - p3 = round(100 * (failure[ddd] / entries[ddd])) - p4 = round(100 * (failure[ddd] + badred[ddd] + reor1[ddd]) / entries[ddd]) - - output(f"// Département {ddd}") - output(f"// {entries[ddd]} Entrées") - output(f"// {diploma[ddd]} Diplômes") - output(f"// {reor2[ddd]} DUT") - output(f"// {p1}-{p2}% de réussite") - output(f"// {goodred[ddd]} Redoublements") - output(f"// {reor1[ddd]} départs de la formation") - output(f"// {badred[ddd]} redoublements autorisés non actés") - output(f"// {failure[ddd]} échecs") - output(f"// {p3}-{p4}% d'échecs") - output(f"// {unknown[ddd]} inconnus") - for x in strangecases: - output(f"// {x}") - - output(f'// orders["{orderkey}"] = {finallayers}') - output(f"// bacs: {bacs}") - # output("\nhttps://observablehq.com/@mbostock/flow-o-matic\n\n") - - # for y in range(4): - # for u in bags[y]: - # for v in bags[y][u]: - # color = "" - # if v[0:4] == "FAIL": - # color = ",red" - # elif v[0:4] == "+DUT": - # color = ",green" - # elif v[0:4] == "DIPL": - # color = ",cyan" - # elif u[0:3] == "EXT": - # color = ",yellow" - # output(f"{u},{v},{bags[y][u][v]}{color}") + posxa = columns[ns["layer"]] + thickness + posxb = columns[targets["layer"]] - thickness + posxc = (3 * posxa + posxb) / 4 + posxd = (posxa + 3 * posxb) / 4 + grad = drawsvg.LinearGradient(posxa, 0, posxb, 0) + grad.add_stop(0, ns["color"], opacity=0.5) + grad.add_stop(1, targets["color"], opacity=0.5) + p = drawsvg.Path(fill=grad, stroke_width=0) + p.M(posxa, link[-2]) + p.C(posxc, link[-2], posxd, target[-2], posxb, target[-2]) + p.L(posxb, target[-1]) + p.C(posxd, target[-1], posxc, link[-1], posxa, link[-1]) + p.Z() + g2.append(p) + d.append(g2) + d.append(g1) + d.append(g4) + d.append(g3) + d.save_svg(orderkey + ".svg") -printout() +printsvg() +for x in strangecases: + print(f"// {x}")