From b93fd36446b86a06624a5c65b0070ff4da245efc Mon Sep 17 00:00:00 2001 From: Emmanuel Viennet Date: Sat, 14 Aug 2021 18:54:32 +0200 Subject: [PATCH] Script d'import departement ScoDoc7 (ou 8.0) --- README.md | 2 +- app/auth/models.py | 15 + app/models/__init__.py | 4 + app/models/etudiants.py | 4 +- app/models/formsemestre.py | 15 +- app/models/groups.py | 3 +- app/scodoc/notesdb.py | 3 +- app/scodoc/sco_evaluations.py | 7 +- app/scodoc/sco_moduleimpl.py | 13 +- app/scodoc/sco_roles_default.py | 4 + app/utils/__init__.py | 7 - app/views/notes.py | 15 +- scodoc.py | 17 +- tests/conftest.py | 13 +- tools/__init__.py | 9 +- tools/config.sh | 2 +- tools/import_scodoc7_dept.py | 360 ++++++++++++++++++ .../utils => tools}/import_scodoc7_user_db.py | 32 +- tools/migrate_from_scodoc7.sh | 23 +- 19 files changed, 481 insertions(+), 67 deletions(-) delete mode 100644 app/utils/__init__.py create mode 100644 tools/import_scodoc7_dept.py rename {app/utils => tools}/import_scodoc7_user_db.py (66%) diff --git a/README.md b/README.md index b2955bb92..15062b2d6 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ On peut ensuite créer des utilisateurs tests avec: ou mieux, importer les utilisateurs de ScoDoc7 avec: - flask user-db-import-scodoc7 + flask import-scodoc7-users (on peut le faire plus tard avec le script de migration décrit plus bas) (Note: la base `SCOUSERS` de ScoDoc7 n'est pas affectée, ScoDoc8 utilise une base séparée, nommée `SCO8USERS`). diff --git a/app/auth/models.py b/app/auth/models.py index 84b802d18..c0e5f2a5d 100644 --- a/app/auth/models.py +++ b/app/auth/models.py @@ -409,6 +409,21 @@ class UserRole(db.Model): return (role, dept) +def get_super_admin(): + """L'utilisateur admin (où le premier, s'il y en a plusieurs). + Utilisé par les tests unitaires et le script de migration. + """ + admin_role = Role.query.filter_by(name="SuperAdmin").first() + assert admin_role + admin_user = ( + User.query.join(UserRole) + .filter((UserRole.user_id == User.id) & (UserRole.role_id == admin_role.id)) + .first() + ) + assert admin_user + return admin_user + + @login.user_loader def load_user(id): return User.query.get(int(id)) diff --git a/app/models/__init__.py b/app/models/__init__.py index 88fa3774a..6a273257b 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -7,6 +7,7 @@ XXX version préliminaire ScoDoc8 #sco8 sans département CODE_STR_LEN = 16 # chaine pour les codes SHORT_STR_LEN = 32 # courtes chaine, eg acronymes APO_CODE_STR_LEN = 16 # nb de car max d'un code Apogée +GROUPNAME_STR_LEN = 64 from app.models.raw_sql_init import create_database_functions @@ -25,6 +26,7 @@ from app.models.etudiants import ( Admission, ItemSuivi, ItemSuiviTag, + itemsuivi_tags_assoc, EtudAnnotation, ) from app.models.events import Scolog, ScolarNews @@ -34,6 +36,7 @@ from app.models.formations import ( NotesMatiere, NotesModule, NotesTag, + notes_modules_tags, ) from app.models.formsemestre import ( FormSemestre, @@ -43,6 +46,7 @@ from app.models.formsemestre import ( NotesFormsemestreUEComputationExpr, NotesFormsemestreCustomMenu, NotesFormsemestreInscription, + notes_formsemestre_responsables, NotesModuleImpl, notes_modules_enseignants, NotesModuleImplInscription, diff --git a/app/models/etudiants.py b/app/models/etudiants.py index fe1795036..29852ed9d 100644 --- a/app/models/etudiants.py +++ b/app/models/etudiants.py @@ -151,6 +151,6 @@ class EtudAnnotation(db.Model): id = db.Column(db.Integer, primary_key=True) date = db.Column(db.DateTime(timezone=True), server_default=db.func.now()) - etudid = db.Column(db.Integer) # sans contrainte pour garder logs après suppression - author = db.Column(db.Text) # le pseudo (user_name) + etudid = db.Column(db.Integer) # sans contrainte (compat ScoDoc 7)) + author = db.Column(db.Text) # le pseudo (user_name), was zope_authenticated_user comment = db.Column(db.Text) diff --git a/app/models/formsemestre.py b/app/models/formsemestre.py index 0aa3a5b0d..9dc747eb5 100644 --- a/app/models/formsemestre.py +++ b/app/models/formsemestre.py @@ -30,7 +30,9 @@ class FormSemestre(db.Model): etat = db.Column( db.Boolean(), nullable=False, default=True, server_default="true" ) # False si verrouillé - modalite = db.Column(db.String(16), db.ForeignKey("notes_form_modalites.modalite")) + modalite = db.Column( + db.String(SHORT_STR_LEN), db.ForeignKey("notes_form_modalites.modalite") + ) # gestion compensation sem DUT: gestion_compensation = db.Column( db.Boolean(), nullable=False, default=False, server_default="false" @@ -59,10 +61,10 @@ class FormSemestre(db.Model): ens_can_edit_eval = db.Column( db.Boolean(), nullable=False, default=False, server_default="False" ) - # code element semestre Apogee, eg VRTW1 ou V2INCS4,V2INLS4 - elt_sem_apo = db.Column(db.String(APO_CODE_STR_LEN)) - # code element annee Apogee, eg VRT1A ou V2INLA,V2INCA - elt_annee_apo = db.Column(db.String(APO_CODE_STR_LEN)) + # code element semestre Apogee, eg 'VRTW1' ou 'V2INCS4,V2INLS4,...' + elt_sem_apo = db.Column(db.Text()) # peut être fort long ! + # code element annee Apogee, eg 'VRT1A' ou 'V2INLA,V2INCA,...' + elt_annee_apo = db.Column(db.Text()) etapes = db.relationship( "NotesFormsemestreEtape", cascade="all,delete", backref="notes_formsemestre" @@ -109,7 +111,7 @@ class NotesFormModalite(db.Model): id = db.Column(db.Integer, primary_key=True) modalite = db.Column( - db.String(16), + db.String(SHORT_STR_LEN), unique=True, index=True, default=DEFAULT_MODALITE, @@ -255,6 +257,7 @@ notes_modules_enseignants = db.Table( db.ForeignKey("notes_moduleimpl.id"), ), db.Column("ens_id", db.Integer, db.ForeignKey("user.id")), + # ? db.UniqueConstraint("moduleimpl_id", "ens_id"), ) # XXX il manque probablement une relation pour gérer cela diff --git a/app/models/groups.py b/app/models/groups.py index 48e5b38f0..7e330e620 100644 --- a/app/models/groups.py +++ b/app/models/groups.py @@ -8,6 +8,7 @@ from app import db from app.models import APO_CODE_STR_LEN from app.models import SHORT_STR_LEN from app.models import CODE_STR_LEN +from app.models import GROUPNAME_STR_LEN class Partition(db.Model): @@ -54,7 +55,7 @@ class GroupDescr(db.Model): group_id = db.synonym("id") partition_id = db.Column(db.Integer, db.ForeignKey("partition.id")) # "A", "C2", ... (NULL for 'all'): - group_name = db.Column(db.String(SHORT_STR_LEN)) + group_name = db.Column(db.String(GROUPNAME_STR_LEN)) group_membership = db.Table( diff --git a/app/scodoc/notesdb.py b/app/scodoc/notesdb.py index a1c8a54c1..4342d128b 100644 --- a/app/scodoc/notesdb.py +++ b/app/scodoc/notesdb.py @@ -347,7 +347,8 @@ class EditableTable(object): for r in res: self.format_output(r, disable_formatting=disable_formatting) # Add ScoDoc7 id: - r[self.id_name] = r["id"] + if "id" in r: + r[self.id_name] = r["id"] return res def format_output(self, r, disable_formatting=False): diff --git a/app/scodoc/sco_evaluations.py b/app/scodoc/sco_evaluations.py index 5831209cd..cf727909e 100644 --- a/app/scodoc/sco_evaluations.py +++ b/app/scodoc/sco_evaluations.py @@ -677,11 +677,12 @@ def _eval_etat(evals): nb_evals_vides += 1 else: nb_evals_en_cours += 1 - dates.append(e["etat"]["last_modif"]) - - dates = scu.sort_dates(dates) + last_modif = e["etat"]["last_modif"] + if last_modif is not None: + dates.append(e["etat"]["last_modif"]) if len(dates): + dates = scu.sort_dates(dates) last_modif = dates[-1] # date de derniere modif d'une note dans un module else: last_modif = "" diff --git a/app/scodoc/sco_moduleimpl.py b/app/scodoc/sco_moduleimpl.py index 6c65cf097..e26a5439e 100644 --- a/app/scodoc/sco_moduleimpl.py +++ b/app/scodoc/sco_moduleimpl.py @@ -55,8 +55,11 @@ _moduleimplEditor = ndb.EditableTable( _modules_enseignantsEditor = ndb.EditableTable( "notes_modules_enseignants", - "modules_enseignants_id", - ("modules_enseignants_id", "moduleimpl_id", "ens_id"), + None, # pas d'id dans cette Table d'association + ( + "moduleimpl_id", # associe moduleimpl + "ens_id", # a l'id de l'enseignant (User.id) + ), ) @@ -317,12 +320,6 @@ def do_ens_create(context, args): return r -def do_ens_delete(context, oid): - "delete ens" - cnx = ndb.GetDBConnexion() - _modules_enseignantsEditor.delete(cnx, oid) - - def can_change_module_resp(context, REQUEST, moduleimpl_id): """Check if current user can modify module resp. (raise exception if not). = Admin, et dir des etud. (si option l'y autorise) diff --git a/app/scodoc/sco_roles_default.py b/app/scodoc/sco_roles_default.py index 505c95e7d..0670bc806 100644 --- a/app/scodoc/sco_roles_default.py +++ b/app/scodoc/sco_roles_default.py @@ -52,6 +52,10 @@ SCO_ROLES_DEFAULTS = { p.ScoUsersAdmin, p.ScoChangePreferences, ), + # RespPE est le responsable poursuites d'études + # il peut ajouter des tags sur les formations: + # (doit avoir un rôle Ens en plus !) + "RespPe": (p.ScoEditFormationTags,), # Super Admin est un root: création/suppression de départements # _tous_ les droits # Afin d'avoir tous les droits, il ne doit pas être asscoié à un département diff --git a/app/utils/__init__.py b/app/utils/__init__.py deleted file mode 100644 index 8d0629d2d..000000000 --- a/app/utils/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# -*- mode: python -*- -# -*- coding: utf-8 -*- - -# Utilitaires divers, à utiliser en ligne de commande -# via flask - -from app.utils.import_scodoc7_user_db import import_scodoc7_user_db diff --git a/app/views/notes.py b/app/views/notes.py index 3d41ea05e..198c5c2e4 100644 --- a/app/views/notes.py +++ b/app/views/notes.py @@ -1210,8 +1210,11 @@ def formsemestre_enseignants_list(context, REQUEST, formsemestre_id, format="htm @scodoc @permission_required(Permission.ScoView) @scodoc7func(context) -def edit_enseignants_form_delete(context, REQUEST, moduleimpl_id, ens_id): - "remove ens" +def edit_enseignants_form_delete(context, REQUEST, moduleimpl_id, ens_id: int): + """remove ens from this modueimpl + + ens_id: user.id + """ M, _ = sco_moduleimpl.can_change_ens(context, REQUEST, moduleimpl_id) # search ens_id ok = False @@ -1221,7 +1224,13 @@ def edit_enseignants_form_delete(context, REQUEST, moduleimpl_id, ens_id): break if not ok: raise ScoValueError("invalid ens_id (%s)" % ens_id) - sco_moduleimpl.do_ens_delete(context, ens["modules_enseignants_id"]) + ndb.SimpleQuery( + """DELETE FROM notes_modules_enseignants + WHERE moduleimpl_id = %(moduleimpl_id)s + AND ens_id = %(ens_id)s + """, + {"module_impl_id": moduleimpl_id, "ens_id": ens_id}, + ) return flask.redirect("edit_enseignants_form?moduleimpl_id=%s" % moduleimpl_id) diff --git a/scodoc.py b/scodoc.py index c1e7a60da..89cbd5f36 100755 --- a/scodoc.py +++ b/scodoc.py @@ -24,7 +24,7 @@ from app.auth.models import User, Role, UserRole from app import models from app.views import notes, scolar, absences -import app.utils as utils +import tools from config import DevConfig @@ -195,13 +195,24 @@ def test_interactive(filename=None): @app.cli.command() @with_appcontext -def user_db_import_scodoc7(): # user-db-import-scodoc7 +def import_scodoc7_users(): # import-scodoc7-users """Import used defined in ScoDoc7 postgresql database into ScoDoc8 The old database SCOUSERS must be alive and readable by the current user. This script is typically run as unix user "scodoc". The original SCOUSERS database is left unmodified. """ - utils.import_scodoc7_user_db() + messages = tools.import_scodoc7_user_db() + click.echo("----") + click.echo(f"import terminé: {len(messages)} warnings\n") + click.echo("\n".join(messages) + "\n") + + +@app.cli.command() +@click.argument("dept") +@with_appcontext +def import_scodoc7_dept(dept): # import-scodoc7-dept + """Import département ScoDoc7""" + tools.import_scodoc7_dept(dept) @app.cli.command() diff --git a/tests/conftest.py b/tests/conftest.py index f2de28208..a3f1e9c79 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,7 @@ from app import db, create_app from app import initialize_scodoc_database, clear_scodoc_cache from app import models from app.auth.models import User, Role, UserRole, Permission +from app.auth.models import get_super_admin from app.scodoc import sco_bulletins_standard from app.scodoc import notesdb as ndb @@ -24,17 +25,7 @@ def test_client(): # erase and reset database: initialize_scodoc_database(erase=True) # Loge l'utilisateur super-admin - admin_role = Role.query.filter_by(name="SuperAdmin").first() - assert admin_role - admin_user = ( - User.query.join(UserRole) - .filter( - (UserRole.user_id == User.id) - & (UserRole.role_id == admin_role.id) - ) - .first() - ) - assert admin_user + admin_user = get_super_admin() login_user(admin_user) # Vérifie que l'utilisateur "bach" existe u = User.query.filter_by(user_name="bach").first() diff --git a/tools/__init__.py b/tools/__init__.py index b2501688f..cf9debf63 100644 --- a/tools/__init__.py +++ b/tools/__init__.py @@ -1 +1,8 @@ -# tools package: en cours de restructuration +# -*- mode: python -*- +# -*- coding: utf-8 -*- + +# Utilitaires divers, à utiliser en ligne de commande +# via flask + +from tools.import_scodoc7_user_db import import_scodoc7_user_db +from tools.import_scodoc7_dept import import_scodoc7_dept diff --git a/tools/config.sh b/tools/config.sh index 9cb70f2ca..82ac157b6 100644 --- a/tools/config.sh +++ b/tools/config.sh @@ -22,7 +22,7 @@ export SCODOC_VAR_DIR=/opt/scodoc-data export SCODOC_VERSION_DIR="${SCODOC_VAR_DIR}/config/version" export SCODOC_LOGOS_DIR="${SCODOC_VAR_DIR}/config/logos" -# user running ScoDoc server: +# Unix user running ScoDoc server: export SCODOC_USER=scodoc export SCODOC_GROUP=root diff --git a/tools/import_scodoc7_dept.py b/tools/import_scodoc7_dept.py new file mode 100644 index 000000000..dfd6d6033 --- /dev/null +++ b/tools/import_scodoc7_dept.py @@ -0,0 +1,360 @@ +# -*- mode: python -*- +# -*- coding: utf-8 -*- + +import inspect +import pdb + +import psycopg2 +import sqlalchemy +from sqlalchemy import func + +from flask import current_app +from app import db +from app.auth.models import User, get_super_admin +import app +from app import models +from app.scodoc import notesdb as ndb + + +def import_scodoc7_dept(dept_id: str, dept_db_uri=None): + """Importe un département ScoDoc7 dans ScoDoc >= 8.1 + (base de donnée unique) + + Args: + dept_id: acronyme du département ("RT") + dept_db_uri: URI de la base ScoDoc7eg "postgresql:///SCORT" + si None, utilise postgresql:///SCO{dept_id} + """ + dept = models.Departement.query.filter_by(acronym=dept_id).first() + if dept: + raise ValueError(f"le département {dept_id} existe déjà !") + if dept_db_uri is None: + dept_db_uri = f"postgresql:///SCO{dept_id}" + current_app.logger.info(f"connecting to database {dept_db_uri}") + cnx = psycopg2.connect(dept_db_uri) + cursor = cnx.cursor(cursor_factory=ndb.ScoDocCursor) + # Create dept: + dept = models.Departement(acronym=dept_id, description="migré de ScoDoc7") + db.session.add(dept) + db.session.commit() + # + id_from_scodoc7 = {} # { scodoc7id (str) : scodoc8 id (int)} + # Utilisateur de rattachement par défaut: + default_user = get_super_admin() + # + for (table, id_name) in SCO7_TABLES_ORDONNEES: + current_app.logger.info(f"{dept.acronym}: converting {table}...") + klass = get_class_for_table(table) + n = convert_table(dept, cursor, id_from_scodoc7, klass, id_name, default_user) + current_app.logger.info(f" inserted {n} objects.") + + +def get_class_for_table(table): + """Return ScoDoc orm class for the given SQL table: search in our models""" + for name in dir(models): + item = getattr(models, name) + if inspect.isclass(item): + if issubclass(item, db.Model): + if item.__tablename__ == table: + return item + try: # pour les db.Table qui ne sont pas des classes (isclass est faux !) + if item.name == table: + return item + except: + pass + raise ValueError(f"No model for table {table}") + + +def get_boolean_columns(klass): + "return list of names of boolean attributes in this model" + boolean_columns = [] + column_names = sqlalchemy.inspect(klass).columns.keys() + for column_name in column_names: + column = getattr(klass, column_name) + if isinstance(column.expression.type, sqlalchemy.sql.sqltypes.Boolean): + boolean_columns.append(column_name) + return boolean_columns + + +def get_table_max_id(klass): + "return max id in this Table (or -1 if no id)" + if not id in sqlalchemy.inspect(klass).columns.keys(): + return -1 + sql_table = str(klass.description) + con = db.engine.connect() + r = con.execute("SELECT max(id) FROM " + sql_table) + r.fetchone() + if r: + return r[0] + else: # empty table + return 0 + + +def convert_table( + dept, cursor, id_from_scodoc7: dict, klass=None, id_name=None, default_user=None +): + "converti les élements d'une table scodoc7" + # Est-ce une Table ou un Model dans l'ORM ? + if isinstance(klass, sqlalchemy.sql.schema.Table): + is_table = True + current_id = get_table_max_id(klass) + has_id = current_id != -1 + cnx = db.engine.connect() + table_name = str(klass.description) + boolean_columns = [] + else: + is_table = False + has_id = True + cnx = None + table_name = klass.__tablename__ + # Colonnes booléennes (valeurs à convertir depuis int) + boolean_columns = get_boolean_columns(klass) + # Part de l'id le plus haut actuellement présent + # (évidemment, nous sommes les seuls connectés à la base destination !) + current_id = db.session.query(func.max(klass.id)).first() + if (current_id is None) or (current_id[0] is None): + current_id = 0 + else: + current_id = current_id[0] + # mapping: login (scodoc7) : user id (scodoc8) + login2id = {u.user_name: u.id for u in User.query} + + # les tables ont le même nom dans les deux versions de ScoDoc: + cursor.execute(f"SELECT * FROM {table_name}") + objects = cursor.dictfetchall() + + for obj in objects: + current_id += 1 + convert_object( + current_id, + dept, + obj, + has_id, + id_from_scodoc7, + klass, + is_table, + id_name, + boolean_columns, + login2id, + default_user, + cnx, + ) + if cnx: + cnx.close() + + db.session.commit() # écrit la table en une fois + return len(objects) + + +ATTRIBUTES_MAPPING = { + "admissions": { + "debouche": None, + }, + "adresse": { + "entreprise_id": None, + }, + "etud_annotations": { + "zope_authenticated_user": "author", + "zope_remote_addr": None, + }, + "identite": { + "foto": None, + }, + "notes_formsemestre": { + "etape_apo2": None, # => suppressed + "etape_apo3": None, + "etape_apo4": None, + # préférences, plus dans formsemestre: + "bul_show_decision": None, + "bul_show_uevalid": None, + "nomgroupetd": None, + "nomgroupetp": None, + "nomgroupeta": None, + "gestion_absence": None, + "bul_show_codemodules": None, + "bul_show_rangs": None, + "bul_show_ue_rangs": None, + "bul_show_mod_rangs": None, + }, + "partition": { + "compute_ranks": None, + }, + "notes_appreciations": { + "zope_authenticated_user": "author", + "zope_remote_addr": None, + }, + "scolog": { + "remote_addr": None, + "remote_host": None, + }, +} + + +def convert_object( + new_id, + dept, + obj: dict, + has_id: bool = True, + id_from_scodoc7: dict = None, + klass=None, + is_table: bool = False, + id_name=None, + boolean_columns=None, + login2id=None, + default_user=None, + cnx=None, +): + # Supprime l'id ScoDoc7 (eg "formsemestre_id") qui deviendra "id" + if id_name: + old_id = obj[id_name] + del obj[id_name] + else: + old_id = None # tables ScoDoc7 sans id + if is_table: + table_name = str(klass.description) + else: + table_name = klass.__tablename__ + # Les champs contant des id utilisateurs: + # chaine login en ScoDoc7, uid numérique en ScoDoc 8+ + USER_REFS = {"responsable_id", "ens_id", "uid"} + if not is_table: + # Supprime les attributs obsoletes (très anciennes versions de ScoDoc): + attributs = ATTRIBUTES_MAPPING.get(table_name, {}) + # renomme ou supprime les attributs + for k in attributs.keys() & obj.keys(): + v = attributs[k] + if v is not None: + obj[v] = obj[k] + del obj[k] + # map les ids (foreign keys) + for k in obj: + if (k.endswith("id") or k == "object") and k not in USER_REFS | { + "semestre_id", + "sem_id", + }: + old_ref = obj[k] + if old_ref is not None: + if isinstance(old_ref, str): + old_ref = old_ref.strip() + elif k == "entreprise_id": # id numérique spécial + old_ref = f"entreprises.{old_ref}" + elif k == "entreprise_corresp_id": + old_ref = f"entreprise_correspondant.{old_ref}" + + if old_ref == "NULL" or not old_ref: # buggy old entries + new_ref = None + elif old_ref in id_from_scodoc7: + new_ref = id_from_scodoc7[old_ref] + elif (not is_table) and table_name in { + "scolog", + "etud_annotations", + "notes_notes_log", + "scolar_news", + }: + # tables avec "fausses" clés + # (l'object référencé a pu disparaitre) + new_ref = None + else: + raise ValueError(f"no new id for {table_name}.{k}='{obj[k]}' !") + obj[k] = new_ref + # Remape les utilisateur: user.id + # S'il n'existe pas, rattache à l'admin + for k in USER_REFS & obj.keys(): + login_scodoc7 = obj[k] + uid = login2id.get(login_scodoc7) + if not uid: + uid = default_user.id + current_app.logger.warning( + f"non existent user: {login_scodoc7}: giving {table_name}({old_id}) to admin" + ) + # raise ValueError(f"non existent user: {login_scodoc7}") + obj[k] = uid + # Converti les booléens + for k in boolean_columns: + obj[k] = bool(obj[k]) + + # Ajoute le département si besoin: + if hasattr(klass, "dept_id"): + obj["dept_id"] = dept.id + + # Fixe l'id (ainsi nous évitons d'avoir à commit() après chaque entrée) + if has_id: + obj["id"] = new_id + + if is_table: + statement = sqlalchemy.insert(klass).values(**obj) + _ = cnx.execute(statement) + else: + new_obj = klass(**obj) # ORM object + db.session.add(new_obj) + + # Stocke l'id pour les références (foreign keys): + if id_name and has_id: + if isinstance(old_id, int): + # les id int étaient utilisés pour les "entreprises" + old_id = table_name + "." + str(old_id) + id_from_scodoc7[old_id] = new_id + + +# tables ordonnées topologiquement pour les clés étrangères: +# g = nx.read_adjlist("misc/model-scodoc7.csv", create_using=nx.DiGraph,delimiter=";") +# L = list(reversed(list(nx.topological_sort(g)))) +SCO7_TABLES_ORDONNEES = [ + # (table SQL, nom de l'id scodoc7) + ("notes_formations", "formation_id"), + ("notes_ue", "ue_id"), + ("notes_matieres", "matiere_id"), + ("notes_formsemestre", "formsemestre_id"), + ("notes_modules", "module_id"), + ("notes_moduleimpl", "moduleimpl_id"), + ( + "notes_modules_enseignants", + "modules_enseignants_id", + ), # (relation) avait un id modules_enseignants_id + ("partition", "partition_id"), + ("identite", "etudid"), + ("entreprises", "entreprise_id"), + ("notes_evaluation", "evaluation_id"), + ("group_descr", "group_id"), + ("group_membership", "group_membership_id"), # (relation) + ("notes_semset", "semset_id"), + ("notes_tags", "tag_id"), + ("itemsuivi", "itemsuivi_id"), + ("itemsuivi_tags", "tag_id"), + ("adresse", "adresse_id"), + ("admissions", "adm_id"), + ("absences", ""), + ("scolar_news", "news_id"), + ("scolog", ""), + ("etud_annotations", "id"), + ("billet_absence", "billet_id"), + ("entreprise_correspondant", "entreprise_corresp_id"), + ("entreprise_contact", "entreprise_contact_id"), + ("absences_notifications", ""), + # ("notes_form_modalites", "form_modalite_id"), : déjà initialisées + ("notes_appreciations", "id"), + ("scolar_autorisation_inscription", "autorisation_inscription_id"), + ("scolar_formsemestre_validation", "formsemestre_validation_id"), + ("scolar_events", "event_id"), + ("notes_notes_log", "id"), + ("notes_notes", ""), + ("notes_moduleimpl_inscription", "moduleimpl_inscription_id"), + ("notes_formsemestre_inscription", "formsemestre_inscription_id"), + ("notes_formsemestre_custommenu", "custommenu_id"), + ( + "notes_formsemestre_ue_computation_expr", + "notes_formsemestre_ue_computation_expr_id", + ), + ("notes_formsemestre_uecoef", "formsemestre_uecoef_id"), + ("notes_semset_formsemestre", ""), # (relation) + ("notes_formsemestre_etapes", ""), + ("notes_formsemestre_responsables", ""), # (relation) + ("notes_modules_tags", ""), + ("itemsuivi_tags_assoc", ""), # (relation) + ("sco_prefs", "pref_id"), +] + +""" +from tools.import_scodoc7_dept import * +import_scodoc7_dept( "RT" ) +""" \ No newline at end of file diff --git a/app/utils/import_scodoc7_user_db.py b/tools/import_scodoc7_user_db.py similarity index 66% rename from app/utils/import_scodoc7_user_db.py rename to tools/import_scodoc7_user_db.py index 034d4d68f..39827e874 100644 --- a/app/utils/import_scodoc7_user_db.py +++ b/tools/import_scodoc7_user_db.py @@ -17,6 +17,7 @@ def import_scodoc7_user_db(scodoc7_db="dbname=SCOUSERS"): The resulting users are in SCO8USERS, handled via Flask/SQLAlchemy ORM. """ + messages = [] cnx = psycopg2.connect(scodoc7_db) cursor = cnx.cursor(cursor_factory=psycopg2.extras.DictCursor) cursor.execute("SELECT * FROM sco_users;") @@ -46,27 +47,34 @@ def import_scodoc7_user_db(scodoc7_db="dbname=SCOUSERS"): else: roles7 = [] for role_dept in roles7: - m = re.match(r"^-?([A-Za-z0-9]+?)([A-Z][A-Za-z0-9]*?)$", role_dept) + # Cas particulier RespPeRT + m = re.match(r"^(-?RespPe)([A-Z][A-Za-z0-9]*?)$", role_dept) if not m: - current_app.logger.warning( - "User {}: ignoring role {}".format(u7["user_name"], role_dept) + # Cas général: eg EnsRT + m = re.match(r"^(-?[A-Za-z0-9]+?)([A-Z][A-Za-z0-9]*?)$", role_dept) + if not m: + msg = ( + f"User {u7['user_name']}: invalid role '{role_dept}' (ignoring)" ) + current_app.logger.warning(msg) + messages.append(msg) else: role_name = m.group(1) if role_name.startswith("-"): # disabled users in ScoDoc7 role_name = role_name[1:] assert not u.active - dept = m.group(2) - role = Role.query.filter_by(name=role_name).first() - if not role: - current_app.logger.warning( - "User {}: ignoring role {}".format( - u7["user_name"], role_dept - ) - ) + # silently ignore old (disabled) role else: - u.add_role(role, dept) + dept = m.group(2) + role = Role.query.filter_by(name=role_name).first() + if not role: + msg = f"User {u7['user_name']}: ignoring role '{role_dept}'" + current_app.logger.warning(msg) + messages.append(msg) + else: + u.add_role(role, dept) db.session.add(u) current_app.logger.info("imported user {}".format(u)) db.session.commit() + return messages \ No newline at end of file diff --git a/tools/migrate_from_scodoc7.sh b/tools/migrate_from_scodoc7.sh index c82694f7a..1ae69971b 100755 --- a/tools/migrate_from_scodoc7.sh +++ b/tools/migrate_from_scodoc7.sh @@ -106,13 +106,22 @@ else fi fi -# Migration base utilisateurs +# ----- Migration base utilisateurs echo -echo "Importer les utilisateurs de ScoDoc7 dans ScoDoc8 ?" +echo "-------------------------------------------------------------" +echo "Importation des utilisateurs de ScoDoc7 dans ScoDoc8 " echo "(la base SCOUSERS de ScoDoc7 sera laissée inchangée)" echo "(les utilisateurs ScoDoc8 existants seront laissés inchangés)" -read -r ans -if [ "$(norm_ans "$ans")" != 'N' ] -then - (cd "$SCODOC_DIR" && flask user-db-import-scodoc7) -fi +echo "-------------------------------------------------------------" +echo + +su -c (cd "$SCODOC_DIR" && flask import-scodoc7-users) + +# ----- Migration bases départements +# les départements ScoDoc7 ont été déplacés dans /opt/scodoc-data/config/dept +for f in "$SCODOC_VAR_DIR"/config/depts/*.cfg +do + dept=$(basename "${f%.*}") + # Liste des bases de données de département: + psql -l | awk '{print $1;}' | grep ^SCO | grep -v SCOUSERS | grep -v SCO8USERS +done \ No newline at end of file