Script d'import departement ScoDoc7 (ou 8.0)

README.md

@@ -145,7 +145,7 @@ On peut ensuite créer des utilisateurs tests avec:
 
 ou mieux, importer les utilisateurs de ScoDoc7 avec:
 
-    flask user-db-import-scodoc7
+    flask import-scodoc7-users
 
 (on peut le faire plus tard avec le script de migration décrit plus bas)
 (Note: la base `SCOUSERS` de ScoDoc7 n'est pas affectée, ScoDoc8 utilise une base séparée, nommée `SCO8USERS`).

app/auth/models.py

@@ -409,6 +409,21 @@ class UserRole(db.Model):
         return (role, dept)
 
 
+def get_super_admin():
+    """L'utilisateur admin (où le premier, s'il y en a plusieurs).
+    Utilisé par les tests unitaires et le script de migration.
+    """
+    admin_role = Role.query.filter_by(name="SuperAdmin").first()
+    assert admin_role
+    admin_user = (
+        User.query.join(UserRole)
+        .filter((UserRole.user_id == User.id) & (UserRole.role_id == admin_role.id))
+        .first()
+    )
+    assert admin_user
+    return admin_user
+
+
 @login.user_loader
 def load_user(id):
     return User.query.get(int(id))

app/models/__init__.py

@@ -7,6 +7,7 @@ XXX version préliminaire ScoDoc8 #sco8 sans département
 CODE_STR_LEN = 16  # chaine pour les codes
 SHORT_STR_LEN = 32  # courtes chaine, eg acronymes
 APO_CODE_STR_LEN = 16  # nb de car max d'un code Apogée
+GROUPNAME_STR_LEN = 64
 
 from app.models.raw_sql_init import create_database_functions
 
@@ -25,6 +26,7 @@ from app.models.etudiants import (
     Admission,
     ItemSuivi,
     ItemSuiviTag,
+    itemsuivi_tags_assoc,
     EtudAnnotation,
 )
 from app.models.events import Scolog, ScolarNews
@@ -34,6 +36,7 @@ from app.models.formations import (
     NotesMatiere,
     NotesModule,
     NotesTag,
+    notes_modules_tags,
 )
 from app.models.formsemestre import (
     FormSemestre,
@@ -43,6 +46,7 @@ from app.models.formsemestre import (
     NotesFormsemestreUEComputationExpr,
     NotesFormsemestreCustomMenu,
     NotesFormsemestreInscription,
+    notes_formsemestre_responsables,
     NotesModuleImpl,
     notes_modules_enseignants,
     NotesModuleImplInscription,

app/models/etudiants.py

@@ -151,6 +151,6 @@ class EtudAnnotation(db.Model):
 
     id = db.Column(db.Integer, primary_key=True)
     date = db.Column(db.DateTime(timezone=True), server_default=db.func.now())
-    etudid = db.Column(db.Integer)  # sans contrainte pour garder logs après suppression
+    etudid = db.Column(db.Integer)  # sans contrainte (compat ScoDoc 7))
-    author = db.Column(db.Text)  # le pseudo (user_name)
+    author = db.Column(db.Text)  # le pseudo (user_name), was zope_authenticated_user
     comment = db.Column(db.Text)

app/models/formsemestre.py

@@ -30,7 +30,9 @@ class FormSemestre(db.Model):
     etat = db.Column(
         db.Boolean(), nullable=False, default=True, server_default="true"
     )  # False si verrouillé
-    modalite = db.Column(db.String(16), db.ForeignKey("notes_form_modalites.modalite"))
+    modalite = db.Column(
+        db.String(SHORT_STR_LEN), db.ForeignKey("notes_form_modalites.modalite")
+    )
     # gestion compensation sem DUT:
     gestion_compensation = db.Column(
         db.Boolean(), nullable=False, default=False, server_default="false"
@@ -59,10 +61,10 @@ class FormSemestre(db.Model):
     ens_can_edit_eval = db.Column(
         db.Boolean(), nullable=False, default=False, server_default="False"
     )
-    # code element semestre Apogee, eg VRTW1 ou V2INCS4,V2INLS4
+    # code element semestre Apogee, eg 'VRTW1' ou 'V2INCS4,V2INLS4,...'
-    elt_sem_apo = db.Column(db.String(APO_CODE_STR_LEN))
+    elt_sem_apo = db.Column(db.Text())  # peut être fort long !
-    # code element annee Apogee, eg VRT1A ou V2INLA,V2INCA
+    # code element annee Apogee, eg 'VRT1A' ou 'V2INLA,V2INCA,...'
-    elt_annee_apo = db.Column(db.String(APO_CODE_STR_LEN))
+    elt_annee_apo = db.Column(db.Text())
 
     etapes = db.relationship(
         "NotesFormsemestreEtape", cascade="all,delete", backref="notes_formsemestre"
@@ -109,7 +111,7 @@ class NotesFormModalite(db.Model):
 
     id = db.Column(db.Integer, primary_key=True)
     modalite = db.Column(
-        db.String(16),
+        db.String(SHORT_STR_LEN),
         unique=True,
         index=True,
         default=DEFAULT_MODALITE,
@@ -255,6 +257,7 @@ notes_modules_enseignants = db.Table(
         db.ForeignKey("notes_moduleimpl.id"),
     ),
     db.Column("ens_id", db.Integer, db.ForeignKey("user.id")),
+    # ? db.UniqueConstraint("moduleimpl_id", "ens_id"),
 )
 # XXX il manque probablement une relation pour gérer cela
 

app/models/groups.py

@@ -8,6 +8,7 @@ from app import db
 from app.models import APO_CODE_STR_LEN
 from app.models import SHORT_STR_LEN
 from app.models import CODE_STR_LEN
+from app.models import GROUPNAME_STR_LEN
 
 
 class Partition(db.Model):
@@ -54,7 +55,7 @@ class GroupDescr(db.Model):
     group_id = db.synonym("id")
     partition_id = db.Column(db.Integer, db.ForeignKey("partition.id"))
     #  "A", "C2", ...  (NULL for 'all'):
-    group_name = db.Column(db.String(SHORT_STR_LEN))
+    group_name = db.Column(db.String(GROUPNAME_STR_LEN))
 
 
 group_membership = db.Table(

app/scodoc/notesdb.py

@@ -347,7 +347,8 @@ class EditableTable(object):
         for r in res:
             self.format_output(r, disable_formatting=disable_formatting)
             # Add ScoDoc7 id:
-            r[self.id_name] = r["id"]
+            if "id" in r:
+                r[self.id_name] = r["id"]
         return res
 
     def format_output(self, r, disable_formatting=False):

app/scodoc/sco_evaluations.py

@@ -677,11 +677,12 @@ def _eval_etat(evals):
             nb_evals_vides += 1
         else:
             nb_evals_en_cours += 1
-        dates.append(e["etat"]["last_modif"])
+        last_modif = e["etat"]["last_modif"]
-
+        if last_modif is not None:
-    dates = scu.sort_dates(dates)
+            dates.append(e["etat"]["last_modif"])
 
     if len(dates):
+        dates = scu.sort_dates(dates)
         last_modif = dates[-1]  # date de derniere modif d'une note dans un module
     else:
         last_modif = ""

app/scodoc/sco_moduleimpl.py

@@ -55,8 +55,11 @@ _moduleimplEditor = ndb.EditableTable(
 
 _modules_enseignantsEditor = ndb.EditableTable(
     "notes_modules_enseignants",
-    "modules_enseignants_id",
+    None,  # pas d'id dans cette Table d'association
-    ("modules_enseignants_id", "moduleimpl_id", "ens_id"),
+    (
+        "moduleimpl_id",  # associe moduleimpl
+        "ens_id",  # a l'id de l'enseignant (User.id)
+    ),
 )
 
 
@@ -317,12 +320,6 @@ def do_ens_create(context, args):
     return r
 
 
-def do_ens_delete(context, oid):
-    "delete ens"
-    cnx = ndb.GetDBConnexion()
-    _modules_enseignantsEditor.delete(cnx, oid)
-
-
 def can_change_module_resp(context, REQUEST, moduleimpl_id):
     """Check if current user can modify module resp. (raise exception if not).
     = Admin, et dir des etud. (si option l'y autorise)

app/scodoc/sco_roles_default.py

@@ -52,6 +52,10 @@ SCO_ROLES_DEFAULTS = {
         p.ScoUsersAdmin,
         p.ScoChangePreferences,
     ),
+    # RespPE est le responsable poursuites d'études
+    # il peut ajouter des tags sur les formations:
+    # (doit avoir un rôle Ens en plus !)
+    "RespPe": (p.ScoEditFormationTags,),
     # Super Admin est un root: création/suppression de départements
     # _tous_ les droits
     # Afin d'avoir tous les droits, il ne doit pas être asscoié à un département

app/utils/__init__.py

@@ -1,7 +0,0 @@
-# -*- mode: python -*-
-# -*- coding: utf-8 -*-
-
-# Utilitaires divers, à utiliser en ligne de commande
-# via flask
-
-from app.utils.import_scodoc7_user_db import import_scodoc7_user_db

app/views/notes.py

@@ -1210,8 +1210,11 @@ def formsemestre_enseignants_list(context, REQUEST, formsemestre_id, format="htm
 @scodoc
 @permission_required(Permission.ScoView)
 @scodoc7func(context)
-def edit_enseignants_form_delete(context, REQUEST, moduleimpl_id, ens_id):
+def edit_enseignants_form_delete(context, REQUEST, moduleimpl_id, ens_id: int):
-    "remove ens"
+    """remove ens from this modueimpl
+
+    ens_id:  user.id
+    """
     M, _ = sco_moduleimpl.can_change_ens(context, REQUEST, moduleimpl_id)
     # search ens_id
     ok = False
@@ -1221,7 +1224,13 @@ def edit_enseignants_form_delete(context, REQUEST, moduleimpl_id, ens_id):
             break
     if not ok:
         raise ScoValueError("invalid ens_id (%s)" % ens_id)
-    sco_moduleimpl.do_ens_delete(context, ens["modules_enseignants_id"])
+    ndb.SimpleQuery(
+        """DELETE FROM notes_modules_enseignants
+    WHERE moduleimpl_id = %(moduleimpl_id)s 
+    AND ens_id = %(ens_id)s
+    """,
+        {"module_impl_id": moduleimpl_id, "ens_id": ens_id},
+    )
     return flask.redirect("edit_enseignants_form?moduleimpl_id=%s" % moduleimpl_id)
 
 

scodoc.py

@@ -24,7 +24,7 @@ from app.auth.models import User, Role, UserRole
 from app import models
 
 from app.views import notes, scolar, absences
-import app.utils as utils
+import tools
 
 from config import DevConfig
 
@@ -195,13 +195,24 @@ def test_interactive(filename=None):
 
 @app.cli.command()
 @with_appcontext
-def user_db_import_scodoc7():  # user-db-import-scodoc7
+def import_scodoc7_users():  # import-scodoc7-users
     """Import used defined in ScoDoc7 postgresql database into ScoDoc8
     The old database SCOUSERS  must be alive and readable by the current user.
     This script is typically run as unix user "scodoc".
     The original SCOUSERS database is left unmodified.
     """
-    utils.import_scodoc7_user_db()
+    messages = tools.import_scodoc7_user_db()
+    click.echo("----")
+    click.echo(f"import terminé: {len(messages)} warnings\n")
+    click.echo("\n".join(messages) + "\n")
+
+
+@app.cli.command()
+@click.argument("dept")
+@with_appcontext
+def import_scodoc7_dept(dept):  # import-scodoc7-dept
+    """Import département ScoDoc7"""
+    tools.import_scodoc7_dept(dept)
 
 
 @app.cli.command()

tests/conftest.py

@@ -9,6 +9,7 @@ from app import db, create_app
 from app import initialize_scodoc_database, clear_scodoc_cache
 from app import models
 from app.auth.models import User, Role, UserRole, Permission
+from app.auth.models import get_super_admin
 from app.scodoc import sco_bulletins_standard
 from app.scodoc import notesdb as ndb
 
@@ -24,17 +25,7 @@ def test_client():
                 # erase and reset database:
                 initialize_scodoc_database(erase=True)
                 # Loge l'utilisateur super-admin
-                admin_role = Role.query.filter_by(name="SuperAdmin").first()
+                admin_user = get_super_admin()
-                assert admin_role
-                admin_user = (
-                    User.query.join(UserRole)
-                    .filter(
-                        (UserRole.user_id == User.id)
-                        & (UserRole.role_id == admin_role.id)
-                    )
-                    .first()
-                )
-                assert admin_user
                 login_user(admin_user)
                 # Vérifie que l'utilisateur "bach" existe
                 u = User.query.filter_by(user_name="bach").first()

tools/__init__.py

@@ -1 +1,8 @@
-# tools package: en cours de restructuration
+# -*- mode: python -*-
+# -*- coding: utf-8 -*-
+
+# Utilitaires divers, à utiliser en ligne de commande
+# via flask
+
+from tools.import_scodoc7_user_db import import_scodoc7_user_db
+from tools.import_scodoc7_dept import import_scodoc7_dept

tools/config.sh

@@ -22,7 +22,7 @@ export SCODOC_VAR_DIR=/opt/scodoc-data
 export SCODOC_VERSION_DIR="${SCODOC_VAR_DIR}/config/version"
 export SCODOC_LOGOS_DIR="${SCODOC_VAR_DIR}/config/logos"
 
-# user running ScoDoc server:
+# Unix user running ScoDoc server:
 export SCODOC_USER=scodoc
 export SCODOC_GROUP=root
 

tools/import_scodoc7_dept.py

@@ -0,0 +1,360 @@
+# -*- mode: python -*-
+# -*- coding: utf-8 -*-
+
+import inspect
+import pdb
+
+import psycopg2
+import sqlalchemy
+from sqlalchemy import func
+
+from flask import current_app
+from app import db
+from app.auth.models import User, get_super_admin
+import app
+from app import models
+from app.scodoc import notesdb as ndb
+
+
+def import_scodoc7_dept(dept_id: str, dept_db_uri=None):
+    """Importe un département ScoDoc7 dans ScoDoc >= 8.1
+    (base de donnée unique)
+
+    Args:
+        dept_id: acronyme du département ("RT")
+        dept_db_uri: URI de la base ScoDoc7eg "postgresql:///SCORT"
+        si None, utilise postgresql:///SCO{dept_id}
+    """
+    dept = models.Departement.query.filter_by(acronym=dept_id).first()
+    if dept:
+        raise ValueError(f"le département {dept_id} existe déjà !")
+    if dept_db_uri is None:
+        dept_db_uri = f"postgresql:///SCO{dept_id}"
+    current_app.logger.info(f"connecting to database {dept_db_uri}")
+    cnx = psycopg2.connect(dept_db_uri)
+    cursor = cnx.cursor(cursor_factory=ndb.ScoDocCursor)
+    # Create dept:
+    dept = models.Departement(acronym=dept_id, description="migré de ScoDoc7")
+    db.session.add(dept)
+    db.session.commit()
+    #
+    id_from_scodoc7 = {}  # { scodoc7id (str) : scodoc8 id (int)}
+    # Utilisateur de rattachement par défaut:
+    default_user = get_super_admin()
+    #
+    for (table, id_name) in SCO7_TABLES_ORDONNEES:
+        current_app.logger.info(f"{dept.acronym}: converting {table}...")
+        klass = get_class_for_table(table)
+        n = convert_table(dept, cursor, id_from_scodoc7, klass, id_name, default_user)
+        current_app.logger.info(f"     inserted {n} objects.")
+
+
+def get_class_for_table(table):
+    """Return ScoDoc orm class for the given SQL table: search in our models"""
+    for name in dir(models):
+        item = getattr(models, name)
+        if inspect.isclass(item):
+            if issubclass(item, db.Model):
+                if item.__tablename__ == table:
+                    return item
+        try:  # pour les db.Table qui ne sont pas des classes (isclass est faux !)
+            if item.name == table:
+                return item
+        except:
+            pass
+    raise ValueError(f"No model for table {table}")
+
+
+def get_boolean_columns(klass):
+    "return list of names of boolean attributes in this model"
+    boolean_columns = []
+    column_names = sqlalchemy.inspect(klass).columns.keys()
+    for column_name in column_names:
+        column = getattr(klass, column_name)
+        if isinstance(column.expression.type, sqlalchemy.sql.sqltypes.Boolean):
+            boolean_columns.append(column_name)
+    return boolean_columns
+
+
+def get_table_max_id(klass):
+    "return max id in this Table (or -1 if no id)"
+    if not id in sqlalchemy.inspect(klass).columns.keys():
+        return -1
+    sql_table = str(klass.description)
+    con = db.engine.connect()
+    r = con.execute("SELECT max(id) FROM " + sql_table)
+    r.fetchone()
+    if r:
+        return r[0]
+    else:  # empty table
+        return 0
+
+
+def convert_table(
+    dept, cursor, id_from_scodoc7: dict, klass=None, id_name=None, default_user=None
+):
+    "converti les élements d'une table scodoc7"
+    # Est-ce une Table ou un Model dans l'ORM ?
+    if isinstance(klass, sqlalchemy.sql.schema.Table):
+        is_table = True
+        current_id = get_table_max_id(klass)
+        has_id = current_id != -1
+        cnx = db.engine.connect()
+        table_name = str(klass.description)
+        boolean_columns = []
+    else:
+        is_table = False
+        has_id = True
+        cnx = None
+        table_name = klass.__tablename__
+        # Colonnes booléennes (valeurs à convertir depuis int)
+        boolean_columns = get_boolean_columns(klass)
+        # Part de l'id le plus haut actuellement présent
+        # (évidemment, nous sommes les seuls connectés à la base destination !)
+        current_id = db.session.query(func.max(klass.id)).first()
+        if (current_id is None) or (current_id[0] is None):
+            current_id = 0
+        else:
+            current_id = current_id[0]
+    # mapping: login (scodoc7) : user id (scodoc8)
+    login2id = {u.user_name: u.id for u in User.query}
+
+    # les tables ont le même nom dans les deux versions de ScoDoc:
+    cursor.execute(f"SELECT * FROM {table_name}")
+    objects = cursor.dictfetchall()
+
+    for obj in objects:
+        current_id += 1
+        convert_object(
+            current_id,
+            dept,
+            obj,
+            has_id,
+            id_from_scodoc7,
+            klass,
+            is_table,
+            id_name,
+            boolean_columns,
+            login2id,
+            default_user,
+            cnx,
+        )
+    if cnx:
+        cnx.close()
+
+    db.session.commit()  # écrit la table en une fois
+    return len(objects)
+
+
+ATTRIBUTES_MAPPING = {
+    "admissions": {
+        "debouche": None,
+    },
+    "adresse": {
+        "entreprise_id": None,
+    },
+    "etud_annotations": {
+        "zope_authenticated_user": "author",
+        "zope_remote_addr": None,
+    },
+    "identite": {
+        "foto": None,
+    },
+    "notes_formsemestre": {
+        "etape_apo2": None,  # => suppressed
+        "etape_apo3": None,
+        "etape_apo4": None,
+        # préférences, plus dans formsemestre:
+        "bul_show_decision": None,
+        "bul_show_uevalid": None,
+        "nomgroupetd": None,
+        "nomgroupetp": None,
+        "nomgroupeta": None,
+        "gestion_absence": None,
+        "bul_show_codemodules": None,
+        "bul_show_rangs": None,
+        "bul_show_ue_rangs": None,
+        "bul_show_mod_rangs": None,
+    },
+    "partition": {
+        "compute_ranks": None,
+    },
+    "notes_appreciations": {
+        "zope_authenticated_user": "author",
+        "zope_remote_addr": None,
+    },
+    "scolog": {
+        "remote_addr": None,
+        "remote_host": None,
+    },
+}
+
+
+def convert_object(
+    new_id,
+    dept,
+    obj: dict,
+    has_id: bool = True,
+    id_from_scodoc7: dict = None,
+    klass=None,
+    is_table: bool = False,
+    id_name=None,
+    boolean_columns=None,
+    login2id=None,
+    default_user=None,
+    cnx=None,
+):
+    # Supprime l'id ScoDoc7 (eg "formsemestre_id") qui deviendra "id"
+    if id_name:
+        old_id = obj[id_name]
+        del obj[id_name]
+    else:
+        old_id = None  # tables ScoDoc7 sans id
+    if is_table:
+        table_name = str(klass.description)
+    else:
+        table_name = klass.__tablename__
+    # Les champs contant des id utilisateurs:
+    # chaine login en ScoDoc7, uid numérique en ScoDoc 8+
+    USER_REFS = {"responsable_id", "ens_id", "uid"}
+    if not is_table:
+        # Supprime les attributs obsoletes (très anciennes versions de ScoDoc):
+        attributs = ATTRIBUTES_MAPPING.get(table_name, {})
+        # renomme ou supprime les attributs
+        for k in attributs.keys() & obj.keys():
+            v = attributs[k]
+            if v is not None:
+                obj[v] = obj[k]
+            del obj[k]
+    # map les ids (foreign keys)
+    for k in obj:
+        if (k.endswith("id") or k == "object") and k not in USER_REFS | {
+            "semestre_id",
+            "sem_id",
+        }:
+            old_ref = obj[k]
+            if old_ref is not None:
+                if isinstance(old_ref, str):
+                    old_ref = old_ref.strip()
+                elif k == "entreprise_id":  # id numérique spécial
+                    old_ref = f"entreprises.{old_ref}"
+                elif k == "entreprise_corresp_id":
+                    old_ref = f"entreprise_correspondant.{old_ref}"
+
+                if old_ref == "NULL" or not old_ref:  # buggy old entries
+                    new_ref = None
+                elif old_ref in id_from_scodoc7:
+                    new_ref = id_from_scodoc7[old_ref]
+                elif (not is_table) and table_name in {
+                    "scolog",
+                    "etud_annotations",
+                    "notes_notes_log",
+                    "scolar_news",
+                }:
+                    # tables avec "fausses" clés
+                    # (l'object référencé a pu disparaitre)
+                    new_ref = None
+                else:
+                    raise ValueError(f"no new id for {table_name}.{k}='{obj[k]}' !")
+                obj[k] = new_ref
+    # Remape les utilisateur: user.id
+    # S'il n'existe pas, rattache à l'admin
+    for k in USER_REFS & obj.keys():
+        login_scodoc7 = obj[k]
+        uid = login2id.get(login_scodoc7)
+        if not uid:
+            uid = default_user.id
+            current_app.logger.warning(
+                f"non existent user: {login_scodoc7}: giving {table_name}({old_id}) to admin"
+            )
+            # raise ValueError(f"non existent user: {login_scodoc7}")
+        obj[k] = uid
+    # Converti les booléens
+    for k in boolean_columns:
+        obj[k] = bool(obj[k])
+
+    # Ajoute le département si besoin:
+    if hasattr(klass, "dept_id"):
+        obj["dept_id"] = dept.id
+
+    # Fixe l'id (ainsi nous évitons d'avoir à commit() après chaque entrée)
+    if has_id:
+        obj["id"] = new_id
+
+    if is_table:
+        statement = sqlalchemy.insert(klass).values(**obj)
+        _ = cnx.execute(statement)
+    else:
+        new_obj = klass(**obj)  # ORM object
+        db.session.add(new_obj)
+
+    # Stocke l'id pour les références (foreign keys):
+    if id_name and has_id:
+        if isinstance(old_id, int):
+            # les id int étaient utilisés pour les "entreprises"
+            old_id = table_name + "." + str(old_id)
+        id_from_scodoc7[old_id] = new_id
+
+
+# tables ordonnées topologiquement pour les clés étrangères:
+# g = nx.read_adjlist("misc/model-scodoc7.csv", create_using=nx.DiGraph,delimiter=";")
+# L = list(reversed(list(nx.topological_sort(g))))
+SCO7_TABLES_ORDONNEES = [
+    # (table SQL, nom de l'id scodoc7)
+    ("notes_formations", "formation_id"),
+    ("notes_ue", "ue_id"),
+    ("notes_matieres", "matiere_id"),
+    ("notes_formsemestre", "formsemestre_id"),
+    ("notes_modules", "module_id"),
+    ("notes_moduleimpl", "moduleimpl_id"),
+    (
+        "notes_modules_enseignants",
+        "modules_enseignants_id",
+    ),  # (relation) avait un id modules_enseignants_id
+    ("partition", "partition_id"),
+    ("identite", "etudid"),
+    ("entreprises", "entreprise_id"),
+    ("notes_evaluation", "evaluation_id"),
+    ("group_descr", "group_id"),
+    ("group_membership", "group_membership_id"),  # (relation)
+    ("notes_semset", "semset_id"),
+    ("notes_tags", "tag_id"),
+    ("itemsuivi", "itemsuivi_id"),
+    ("itemsuivi_tags", "tag_id"),
+    ("adresse", "adresse_id"),
+    ("admissions", "adm_id"),
+    ("absences", ""),
+    ("scolar_news", "news_id"),
+    ("scolog", ""),
+    ("etud_annotations", "id"),
+    ("billet_absence", "billet_id"),
+    ("entreprise_correspondant", "entreprise_corresp_id"),
+    ("entreprise_contact", "entreprise_contact_id"),
+    ("absences_notifications", ""),
+    # ("notes_form_modalites", "form_modalite_id"), : déjà initialisées
+    ("notes_appreciations", "id"),
+    ("scolar_autorisation_inscription", "autorisation_inscription_id"),
+    ("scolar_formsemestre_validation", "formsemestre_validation_id"),
+    ("scolar_events", "event_id"),
+    ("notes_notes_log", "id"),
+    ("notes_notes", ""),
+    ("notes_moduleimpl_inscription", "moduleimpl_inscription_id"),
+    ("notes_formsemestre_inscription", "formsemestre_inscription_id"),
+    ("notes_formsemestre_custommenu", "custommenu_id"),
+    (
+        "notes_formsemestre_ue_computation_expr",
+        "notes_formsemestre_ue_computation_expr_id",
+    ),
+    ("notes_formsemestre_uecoef", "formsemestre_uecoef_id"),
+    ("notes_semset_formsemestre", ""),  # (relation)
+    ("notes_formsemestre_etapes", ""),
+    ("notes_formsemestre_responsables", ""),  # (relation)
+    ("notes_modules_tags", ""),
+    ("itemsuivi_tags_assoc", ""),  # (relation)
+    ("sco_prefs", "pref_id"),
+]
+
+"""
+from tools.import_scodoc7_dept import *
+import_scodoc7_dept( "RT" )
+"""

tools/import_scodoc7_user_db.py

@@ -17,6 +17,7 @@ def import_scodoc7_user_db(scodoc7_db="dbname=SCOUSERS"):
     The resulting users are in SCO8USERS,
     handled via Flask/SQLAlchemy ORM.
     """
+    messages = []
     cnx = psycopg2.connect(scodoc7_db)
     cursor = cnx.cursor(cursor_factory=psycopg2.extras.DictCursor)
     cursor.execute("SELECT * FROM sco_users;")
@@ -46,27 +47,34 @@ def import_scodoc7_user_db(scodoc7_db="dbname=SCOUSERS"):
             else:
                 roles7 = []
             for role_dept in roles7:
-                m = re.match(r"^-?([A-Za-z0-9]+?)([A-Z][A-Za-z0-9]*?)$", role_dept)
+                # Cas particulier RespPeRT
+                m = re.match(r"^(-?RespPe)([A-Z][A-Za-z0-9]*?)$", role_dept)
                 if not m:
-                    current_app.logger.warning(
+                    # Cas général: eg EnsRT
-                        "User {}: ignoring role {}".format(u7["user_name"], role_dept)
+                    m = re.match(r"^(-?[A-Za-z0-9]+?)([A-Z][A-Za-z0-9]*?)$", role_dept)
+                if not m:
+                    msg = (
+                        f"User {u7['user_name']}: invalid role '{role_dept}' (ignoring)"
                     )
+                    current_app.logger.warning(msg)
+                    messages.append(msg)
                 else:
                     role_name = m.group(1)
                     if role_name.startswith("-"):
                         # disabled users in ScoDoc7
                         role_name = role_name[1:]
                         assert not u.active
-                    dept = m.group(2)
+                        # silently ignore old (disabled) role
-                    role = Role.query.filter_by(name=role_name).first()
-                    if not role:
-                        current_app.logger.warning(
-                            "User {}: ignoring role {}".format(
-                                u7["user_name"], role_dept
-                            )
-                        )
                     else:
-                        u.add_role(role, dept)
+                        dept = m.group(2)
+                        role = Role.query.filter_by(name=role_name).first()
+                        if not role:
+                            msg = f"User {u7['user_name']}: ignoring role '{role_dept}'"
+                            current_app.logger.warning(msg)
+                            messages.append(msg)
+                        else:
+                            u.add_role(role, dept)
             db.session.add(u)
             current_app.logger.info("imported user {}".format(u))
     db.session.commit()
+    return messages

tools/migrate_from_scodoc7.sh

@@ -106,13 +106,22 @@ else
     fi
 fi
 
-# Migration base utilisateurs
+# ----- Migration base utilisateurs
 echo
-echo "Importer les utilisateurs de ScoDoc7 dans ScoDoc8 ?"
+echo "-------------------------------------------------------------"
+echo "Importation des utilisateurs de ScoDoc7 dans ScoDoc8 "
 echo "(la base SCOUSERS de ScoDoc7 sera laissée inchangée)"
 echo "(les utilisateurs ScoDoc8 existants seront laissés inchangés)"
-read -r ans
+echo "-------------------------------------------------------------"
-if [ "$(norm_ans "$ans")" != 'N' ]
+echo
-then
+
-    (cd "$SCODOC_DIR" && flask user-db-import-scodoc7)
+su -c (cd "$SCODOC_DIR" && flask import-scodoc7-users)
-fi
+
+# ----- Migration bases départements
+# les départements ScoDoc7 ont été déplacés dans /opt/scodoc-data/config/dept
+for f in "$SCODOC_VAR_DIR"/config/depts/*.cfg
+do
+    dept=$(basename "${f%.*}")
+    # Liste des bases de données de département:
+    psql -l | awk '{print $1;}' | grep ^SCO | grep -v SCOUSERS | grep -v SCO8USERS
+done