Script d'import departement ScoDoc7 (ou 8.0)

This commit is contained in:
Emmanuel Viennet 2021-08-14 18:54:32 +02:00
parent 98747c3e8e
commit b93fd36446
19 changed files with 481 additions and 67 deletions

View File

@ -145,7 +145,7 @@ On peut ensuite créer des utilisateurs tests avec:
ou mieux, importer les utilisateurs de ScoDoc7 avec: ou mieux, importer les utilisateurs de ScoDoc7 avec:
flask user-db-import-scodoc7 flask import-scodoc7-users
(on peut le faire plus tard avec le script de migration décrit plus bas) (on peut le faire plus tard avec le script de migration décrit plus bas)
(Note: la base `SCOUSERS` de ScoDoc7 n'est pas affectée, ScoDoc8 utilise une base séparée, nommée `SCO8USERS`). (Note: la base `SCOUSERS` de ScoDoc7 n'est pas affectée, ScoDoc8 utilise une base séparée, nommée `SCO8USERS`).

View File

@ -409,6 +409,21 @@ class UserRole(db.Model):
return (role, dept) return (role, dept)
def get_super_admin():
"""L'utilisateur admin (où le premier, s'il y en a plusieurs).
Utilisé par les tests unitaires et le script de migration.
"""
admin_role = Role.query.filter_by(name="SuperAdmin").first()
assert admin_role
admin_user = (
User.query.join(UserRole)
.filter((UserRole.user_id == User.id) & (UserRole.role_id == admin_role.id))
.first()
)
assert admin_user
return admin_user
@login.user_loader @login.user_loader
def load_user(id): def load_user(id):
return User.query.get(int(id)) return User.query.get(int(id))

View File

@ -7,6 +7,7 @@ XXX version préliminaire ScoDoc8 #sco8 sans département
CODE_STR_LEN = 16 # chaine pour les codes CODE_STR_LEN = 16 # chaine pour les codes
SHORT_STR_LEN = 32 # courtes chaine, eg acronymes SHORT_STR_LEN = 32 # courtes chaine, eg acronymes
APO_CODE_STR_LEN = 16 # nb de car max d'un code Apogée APO_CODE_STR_LEN = 16 # nb de car max d'un code Apogée
GROUPNAME_STR_LEN = 64
from app.models.raw_sql_init import create_database_functions from app.models.raw_sql_init import create_database_functions
@ -25,6 +26,7 @@ from app.models.etudiants import (
Admission, Admission,
ItemSuivi, ItemSuivi,
ItemSuiviTag, ItemSuiviTag,
itemsuivi_tags_assoc,
EtudAnnotation, EtudAnnotation,
) )
from app.models.events import Scolog, ScolarNews from app.models.events import Scolog, ScolarNews
@ -34,6 +36,7 @@ from app.models.formations import (
NotesMatiere, NotesMatiere,
NotesModule, NotesModule,
NotesTag, NotesTag,
notes_modules_tags,
) )
from app.models.formsemestre import ( from app.models.formsemestre import (
FormSemestre, FormSemestre,
@ -43,6 +46,7 @@ from app.models.formsemestre import (
NotesFormsemestreUEComputationExpr, NotesFormsemestreUEComputationExpr,
NotesFormsemestreCustomMenu, NotesFormsemestreCustomMenu,
NotesFormsemestreInscription, NotesFormsemestreInscription,
notes_formsemestre_responsables,
NotesModuleImpl, NotesModuleImpl,
notes_modules_enseignants, notes_modules_enseignants,
NotesModuleImplInscription, NotesModuleImplInscription,

View File

@ -151,6 +151,6 @@ class EtudAnnotation(db.Model):
id = db.Column(db.Integer, primary_key=True) id = db.Column(db.Integer, primary_key=True)
date = db.Column(db.DateTime(timezone=True), server_default=db.func.now()) date = db.Column(db.DateTime(timezone=True), server_default=db.func.now())
etudid = db.Column(db.Integer) # sans contrainte pour garder logs après suppression etudid = db.Column(db.Integer) # sans contrainte (compat ScoDoc 7))
author = db.Column(db.Text) # le pseudo (user_name) author = db.Column(db.Text) # le pseudo (user_name), was zope_authenticated_user
comment = db.Column(db.Text) comment = db.Column(db.Text)

View File

@ -30,7 +30,9 @@ class FormSemestre(db.Model):
etat = db.Column( etat = db.Column(
db.Boolean(), nullable=False, default=True, server_default="true" db.Boolean(), nullable=False, default=True, server_default="true"
) # False si verrouillé ) # False si verrouillé
modalite = db.Column(db.String(16), db.ForeignKey("notes_form_modalites.modalite")) modalite = db.Column(
db.String(SHORT_STR_LEN), db.ForeignKey("notes_form_modalites.modalite")
)
# gestion compensation sem DUT: # gestion compensation sem DUT:
gestion_compensation = db.Column( gestion_compensation = db.Column(
db.Boolean(), nullable=False, default=False, server_default="false" db.Boolean(), nullable=False, default=False, server_default="false"
@ -59,10 +61,10 @@ class FormSemestre(db.Model):
ens_can_edit_eval = db.Column( ens_can_edit_eval = db.Column(
db.Boolean(), nullable=False, default=False, server_default="False" db.Boolean(), nullable=False, default=False, server_default="False"
) )
# code element semestre Apogee, eg VRTW1 ou V2INCS4,V2INLS4 # code element semestre Apogee, eg 'VRTW1' ou 'V2INCS4,V2INLS4,...'
elt_sem_apo = db.Column(db.String(APO_CODE_STR_LEN)) elt_sem_apo = db.Column(db.Text()) # peut être fort long !
# code element annee Apogee, eg VRT1A ou V2INLA,V2INCA # code element annee Apogee, eg 'VRT1A' ou 'V2INLA,V2INCA,...'
elt_annee_apo = db.Column(db.String(APO_CODE_STR_LEN)) elt_annee_apo = db.Column(db.Text())
etapes = db.relationship( etapes = db.relationship(
"NotesFormsemestreEtape", cascade="all,delete", backref="notes_formsemestre" "NotesFormsemestreEtape", cascade="all,delete", backref="notes_formsemestre"
@ -109,7 +111,7 @@ class NotesFormModalite(db.Model):
id = db.Column(db.Integer, primary_key=True) id = db.Column(db.Integer, primary_key=True)
modalite = db.Column( modalite = db.Column(
db.String(16), db.String(SHORT_STR_LEN),
unique=True, unique=True,
index=True, index=True,
default=DEFAULT_MODALITE, default=DEFAULT_MODALITE,
@ -255,6 +257,7 @@ notes_modules_enseignants = db.Table(
db.ForeignKey("notes_moduleimpl.id"), db.ForeignKey("notes_moduleimpl.id"),
), ),
db.Column("ens_id", db.Integer, db.ForeignKey("user.id")), db.Column("ens_id", db.Integer, db.ForeignKey("user.id")),
# ? db.UniqueConstraint("moduleimpl_id", "ens_id"),
) )
# XXX il manque probablement une relation pour gérer cela # XXX il manque probablement une relation pour gérer cela

View File

@ -8,6 +8,7 @@ from app import db
from app.models import APO_CODE_STR_LEN from app.models import APO_CODE_STR_LEN
from app.models import SHORT_STR_LEN from app.models import SHORT_STR_LEN
from app.models import CODE_STR_LEN from app.models import CODE_STR_LEN
from app.models import GROUPNAME_STR_LEN
class Partition(db.Model): class Partition(db.Model):
@ -54,7 +55,7 @@ class GroupDescr(db.Model):
group_id = db.synonym("id") group_id = db.synonym("id")
partition_id = db.Column(db.Integer, db.ForeignKey("partition.id")) partition_id = db.Column(db.Integer, db.ForeignKey("partition.id"))
# "A", "C2", ... (NULL for 'all'): # "A", "C2", ... (NULL for 'all'):
group_name = db.Column(db.String(SHORT_STR_LEN)) group_name = db.Column(db.String(GROUPNAME_STR_LEN))
group_membership = db.Table( group_membership = db.Table(

View File

@ -347,7 +347,8 @@ class EditableTable(object):
for r in res: for r in res:
self.format_output(r, disable_formatting=disable_formatting) self.format_output(r, disable_formatting=disable_formatting)
# Add ScoDoc7 id: # Add ScoDoc7 id:
r[self.id_name] = r["id"] if "id" in r:
r[self.id_name] = r["id"]
return res return res
def format_output(self, r, disable_formatting=False): def format_output(self, r, disable_formatting=False):

View File

@ -677,11 +677,12 @@ def _eval_etat(evals):
nb_evals_vides += 1 nb_evals_vides += 1
else: else:
nb_evals_en_cours += 1 nb_evals_en_cours += 1
dates.append(e["etat"]["last_modif"]) last_modif = e["etat"]["last_modif"]
if last_modif is not None:
dates = scu.sort_dates(dates) dates.append(e["etat"]["last_modif"])
if len(dates): if len(dates):
dates = scu.sort_dates(dates)
last_modif = dates[-1] # date de derniere modif d'une note dans un module last_modif = dates[-1] # date de derniere modif d'une note dans un module
else: else:
last_modif = "" last_modif = ""

View File

@ -55,8 +55,11 @@ _moduleimplEditor = ndb.EditableTable(
_modules_enseignantsEditor = ndb.EditableTable( _modules_enseignantsEditor = ndb.EditableTable(
"notes_modules_enseignants", "notes_modules_enseignants",
"modules_enseignants_id", None, # pas d'id dans cette Table d'association
("modules_enseignants_id", "moduleimpl_id", "ens_id"), (
"moduleimpl_id", # associe moduleimpl
"ens_id", # a l'id de l'enseignant (User.id)
),
) )
@ -317,12 +320,6 @@ def do_ens_create(context, args):
return r return r
def do_ens_delete(context, oid):
"delete ens"
cnx = ndb.GetDBConnexion()
_modules_enseignantsEditor.delete(cnx, oid)
def can_change_module_resp(context, REQUEST, moduleimpl_id): def can_change_module_resp(context, REQUEST, moduleimpl_id):
"""Check if current user can modify module resp. (raise exception if not). """Check if current user can modify module resp. (raise exception if not).
= Admin, et dir des etud. (si option l'y autorise) = Admin, et dir des etud. (si option l'y autorise)

View File

@ -52,6 +52,10 @@ SCO_ROLES_DEFAULTS = {
p.ScoUsersAdmin, p.ScoUsersAdmin,
p.ScoChangePreferences, p.ScoChangePreferences,
), ),
# RespPE est le responsable poursuites d'études
# il peut ajouter des tags sur les formations:
# (doit avoir un rôle Ens en plus !)
"RespPe": (p.ScoEditFormationTags,),
# Super Admin est un root: création/suppression de départements # Super Admin est un root: création/suppression de départements
# _tous_ les droits # _tous_ les droits
# Afin d'avoir tous les droits, il ne doit pas être asscoié à un département # Afin d'avoir tous les droits, il ne doit pas être asscoié à un département

View File

@ -1,7 +0,0 @@
# -*- mode: python -*-
# -*- coding: utf-8 -*-
# Utilitaires divers, à utiliser en ligne de commande
# via flask
from app.utils.import_scodoc7_user_db import import_scodoc7_user_db

View File

@ -1210,8 +1210,11 @@ def formsemestre_enseignants_list(context, REQUEST, formsemestre_id, format="htm
@scodoc @scodoc
@permission_required(Permission.ScoView) @permission_required(Permission.ScoView)
@scodoc7func(context) @scodoc7func(context)
def edit_enseignants_form_delete(context, REQUEST, moduleimpl_id, ens_id): def edit_enseignants_form_delete(context, REQUEST, moduleimpl_id, ens_id: int):
"remove ens" """remove ens from this modueimpl
ens_id: user.id
"""
M, _ = sco_moduleimpl.can_change_ens(context, REQUEST, moduleimpl_id) M, _ = sco_moduleimpl.can_change_ens(context, REQUEST, moduleimpl_id)
# search ens_id # search ens_id
ok = False ok = False
@ -1221,7 +1224,13 @@ def edit_enseignants_form_delete(context, REQUEST, moduleimpl_id, ens_id):
break break
if not ok: if not ok:
raise ScoValueError("invalid ens_id (%s)" % ens_id) raise ScoValueError("invalid ens_id (%s)" % ens_id)
sco_moduleimpl.do_ens_delete(context, ens["modules_enseignants_id"]) ndb.SimpleQuery(
"""DELETE FROM notes_modules_enseignants
WHERE moduleimpl_id = %(moduleimpl_id)s
AND ens_id = %(ens_id)s
""",
{"module_impl_id": moduleimpl_id, "ens_id": ens_id},
)
return flask.redirect("edit_enseignants_form?moduleimpl_id=%s" % moduleimpl_id) return flask.redirect("edit_enseignants_form?moduleimpl_id=%s" % moduleimpl_id)

View File

@ -24,7 +24,7 @@ from app.auth.models import User, Role, UserRole
from app import models from app import models
from app.views import notes, scolar, absences from app.views import notes, scolar, absences
import app.utils as utils import tools
from config import DevConfig from config import DevConfig
@ -195,13 +195,24 @@ def test_interactive(filename=None):
@app.cli.command() @app.cli.command()
@with_appcontext @with_appcontext
def user_db_import_scodoc7(): # user-db-import-scodoc7 def import_scodoc7_users(): # import-scodoc7-users
"""Import used defined in ScoDoc7 postgresql database into ScoDoc8 """Import used defined in ScoDoc7 postgresql database into ScoDoc8
The old database SCOUSERS must be alive and readable by the current user. The old database SCOUSERS must be alive and readable by the current user.
This script is typically run as unix user "scodoc". This script is typically run as unix user "scodoc".
The original SCOUSERS database is left unmodified. The original SCOUSERS database is left unmodified.
""" """
utils.import_scodoc7_user_db() messages = tools.import_scodoc7_user_db()
click.echo("----")
click.echo(f"import terminé: {len(messages)} warnings\n")
click.echo("\n".join(messages) + "\n")
@app.cli.command()
@click.argument("dept")
@with_appcontext
def import_scodoc7_dept(dept): # import-scodoc7-dept
"""Import département ScoDoc7"""
tools.import_scodoc7_dept(dept)
@app.cli.command() @app.cli.command()

View File

@ -9,6 +9,7 @@ from app import db, create_app
from app import initialize_scodoc_database, clear_scodoc_cache from app import initialize_scodoc_database, clear_scodoc_cache
from app import models from app import models
from app.auth.models import User, Role, UserRole, Permission from app.auth.models import User, Role, UserRole, Permission
from app.auth.models import get_super_admin
from app.scodoc import sco_bulletins_standard from app.scodoc import sco_bulletins_standard
from app.scodoc import notesdb as ndb from app.scodoc import notesdb as ndb
@ -24,17 +25,7 @@ def test_client():
# erase and reset database: # erase and reset database:
initialize_scodoc_database(erase=True) initialize_scodoc_database(erase=True)
# Loge l'utilisateur super-admin # Loge l'utilisateur super-admin
admin_role = Role.query.filter_by(name="SuperAdmin").first() admin_user = get_super_admin()
assert admin_role
admin_user = (
User.query.join(UserRole)
.filter(
(UserRole.user_id == User.id)
& (UserRole.role_id == admin_role.id)
)
.first()
)
assert admin_user
login_user(admin_user) login_user(admin_user)
# Vérifie que l'utilisateur "bach" existe # Vérifie que l'utilisateur "bach" existe
u = User.query.filter_by(user_name="bach").first() u = User.query.filter_by(user_name="bach").first()

View File

@ -1 +1,8 @@
# tools package: en cours de restructuration # -*- mode: python -*-
# -*- coding: utf-8 -*-
# Utilitaires divers, à utiliser en ligne de commande
# via flask
from tools.import_scodoc7_user_db import import_scodoc7_user_db
from tools.import_scodoc7_dept import import_scodoc7_dept

View File

@ -22,7 +22,7 @@ export SCODOC_VAR_DIR=/opt/scodoc-data
export SCODOC_VERSION_DIR="${SCODOC_VAR_DIR}/config/version" export SCODOC_VERSION_DIR="${SCODOC_VAR_DIR}/config/version"
export SCODOC_LOGOS_DIR="${SCODOC_VAR_DIR}/config/logos" export SCODOC_LOGOS_DIR="${SCODOC_VAR_DIR}/config/logos"
# user running ScoDoc server: # Unix user running ScoDoc server:
export SCODOC_USER=scodoc export SCODOC_USER=scodoc
export SCODOC_GROUP=root export SCODOC_GROUP=root

View File

@ -0,0 +1,360 @@
# -*- mode: python -*-
# -*- coding: utf-8 -*-
import inspect
import pdb
import psycopg2
import sqlalchemy
from sqlalchemy import func
from flask import current_app
from app import db
from app.auth.models import User, get_super_admin
import app
from app import models
from app.scodoc import notesdb as ndb
def import_scodoc7_dept(dept_id: str, dept_db_uri=None):
"""Importe un département ScoDoc7 dans ScoDoc >= 8.1
(base de donnée unique)
Args:
dept_id: acronyme du département ("RT")
dept_db_uri: URI de la base ScoDoc7eg "postgresql:///SCORT"
si None, utilise postgresql:///SCO{dept_id}
"""
dept = models.Departement.query.filter_by(acronym=dept_id).first()
if dept:
raise ValueError(f"le département {dept_id} existe déjà !")
if dept_db_uri is None:
dept_db_uri = f"postgresql:///SCO{dept_id}"
current_app.logger.info(f"connecting to database {dept_db_uri}")
cnx = psycopg2.connect(dept_db_uri)
cursor = cnx.cursor(cursor_factory=ndb.ScoDocCursor)
# Create dept:
dept = models.Departement(acronym=dept_id, description="migré de ScoDoc7")
db.session.add(dept)
db.session.commit()
#
id_from_scodoc7 = {} # { scodoc7id (str) : scodoc8 id (int)}
# Utilisateur de rattachement par défaut:
default_user = get_super_admin()
#
for (table, id_name) in SCO7_TABLES_ORDONNEES:
current_app.logger.info(f"{dept.acronym}: converting {table}...")
klass = get_class_for_table(table)
n = convert_table(dept, cursor, id_from_scodoc7, klass, id_name, default_user)
current_app.logger.info(f" inserted {n} objects.")
def get_class_for_table(table):
"""Return ScoDoc orm class for the given SQL table: search in our models"""
for name in dir(models):
item = getattr(models, name)
if inspect.isclass(item):
if issubclass(item, db.Model):
if item.__tablename__ == table:
return item
try: # pour les db.Table qui ne sont pas des classes (isclass est faux !)
if item.name == table:
return item
except:
pass
raise ValueError(f"No model for table {table}")
def get_boolean_columns(klass):
"return list of names of boolean attributes in this model"
boolean_columns = []
column_names = sqlalchemy.inspect(klass).columns.keys()
for column_name in column_names:
column = getattr(klass, column_name)
if isinstance(column.expression.type, sqlalchemy.sql.sqltypes.Boolean):
boolean_columns.append(column_name)
return boolean_columns
def get_table_max_id(klass):
"return max id in this Table (or -1 if no id)"
if not id in sqlalchemy.inspect(klass).columns.keys():
return -1
sql_table = str(klass.description)
con = db.engine.connect()
r = con.execute("SELECT max(id) FROM " + sql_table)
r.fetchone()
if r:
return r[0]
else: # empty table
return 0
def convert_table(
dept, cursor, id_from_scodoc7: dict, klass=None, id_name=None, default_user=None
):
"converti les élements d'une table scodoc7"
# Est-ce une Table ou un Model dans l'ORM ?
if isinstance(klass, sqlalchemy.sql.schema.Table):
is_table = True
current_id = get_table_max_id(klass)
has_id = current_id != -1
cnx = db.engine.connect()
table_name = str(klass.description)
boolean_columns = []
else:
is_table = False
has_id = True
cnx = None
table_name = klass.__tablename__
# Colonnes booléennes (valeurs à convertir depuis int)
boolean_columns = get_boolean_columns(klass)
# Part de l'id le plus haut actuellement présent
# (évidemment, nous sommes les seuls connectés à la base destination !)
current_id = db.session.query(func.max(klass.id)).first()
if (current_id is None) or (current_id[0] is None):
current_id = 0
else:
current_id = current_id[0]
# mapping: login (scodoc7) : user id (scodoc8)
login2id = {u.user_name: u.id for u in User.query}
# les tables ont le même nom dans les deux versions de ScoDoc:
cursor.execute(f"SELECT * FROM {table_name}")
objects = cursor.dictfetchall()
for obj in objects:
current_id += 1
convert_object(
current_id,
dept,
obj,
has_id,
id_from_scodoc7,
klass,
is_table,
id_name,
boolean_columns,
login2id,
default_user,
cnx,
)
if cnx:
cnx.close()
db.session.commit() # écrit la table en une fois
return len(objects)
ATTRIBUTES_MAPPING = {
"admissions": {
"debouche": None,
},
"adresse": {
"entreprise_id": None,
},
"etud_annotations": {
"zope_authenticated_user": "author",
"zope_remote_addr": None,
},
"identite": {
"foto": None,
},
"notes_formsemestre": {
"etape_apo2": None, # => suppressed
"etape_apo3": None,
"etape_apo4": None,
# préférences, plus dans formsemestre:
"bul_show_decision": None,
"bul_show_uevalid": None,
"nomgroupetd": None,
"nomgroupetp": None,
"nomgroupeta": None,
"gestion_absence": None,
"bul_show_codemodules": None,
"bul_show_rangs": None,
"bul_show_ue_rangs": None,
"bul_show_mod_rangs": None,
},
"partition": {
"compute_ranks": None,
},
"notes_appreciations": {
"zope_authenticated_user": "author",
"zope_remote_addr": None,
},
"scolog": {
"remote_addr": None,
"remote_host": None,
},
}
def convert_object(
new_id,
dept,
obj: dict,
has_id: bool = True,
id_from_scodoc7: dict = None,
klass=None,
is_table: bool = False,
id_name=None,
boolean_columns=None,
login2id=None,
default_user=None,
cnx=None,
):
# Supprime l'id ScoDoc7 (eg "formsemestre_id") qui deviendra "id"
if id_name:
old_id = obj[id_name]
del obj[id_name]
else:
old_id = None # tables ScoDoc7 sans id
if is_table:
table_name = str(klass.description)
else:
table_name = klass.__tablename__
# Les champs contant des id utilisateurs:
# chaine login en ScoDoc7, uid numérique en ScoDoc 8+
USER_REFS = {"responsable_id", "ens_id", "uid"}
if not is_table:
# Supprime les attributs obsoletes (très anciennes versions de ScoDoc):
attributs = ATTRIBUTES_MAPPING.get(table_name, {})
# renomme ou supprime les attributs
for k in attributs.keys() & obj.keys():
v = attributs[k]
if v is not None:
obj[v] = obj[k]
del obj[k]
# map les ids (foreign keys)
for k in obj:
if (k.endswith("id") or k == "object") and k not in USER_REFS | {
"semestre_id",
"sem_id",
}:
old_ref = obj[k]
if old_ref is not None:
if isinstance(old_ref, str):
old_ref = old_ref.strip()
elif k == "entreprise_id": # id numérique spécial
old_ref = f"entreprises.{old_ref}"
elif k == "entreprise_corresp_id":
old_ref = f"entreprise_correspondant.{old_ref}"
if old_ref == "NULL" or not old_ref: # buggy old entries
new_ref = None
elif old_ref in id_from_scodoc7:
new_ref = id_from_scodoc7[old_ref]
elif (not is_table) and table_name in {
"scolog",
"etud_annotations",
"notes_notes_log",
"scolar_news",
}:
# tables avec "fausses" clés
# (l'object référencé a pu disparaitre)
new_ref = None
else:
raise ValueError(f"no new id for {table_name}.{k}='{obj[k]}' !")
obj[k] = new_ref
# Remape les utilisateur: user.id
# S'il n'existe pas, rattache à l'admin
for k in USER_REFS & obj.keys():
login_scodoc7 = obj[k]
uid = login2id.get(login_scodoc7)
if not uid:
uid = default_user.id
current_app.logger.warning(
f"non existent user: {login_scodoc7}: giving {table_name}({old_id}) to admin"
)
# raise ValueError(f"non existent user: {login_scodoc7}")
obj[k] = uid
# Converti les booléens
for k in boolean_columns:
obj[k] = bool(obj[k])
# Ajoute le département si besoin:
if hasattr(klass, "dept_id"):
obj["dept_id"] = dept.id
# Fixe l'id (ainsi nous évitons d'avoir à commit() après chaque entrée)
if has_id:
obj["id"] = new_id
if is_table:
statement = sqlalchemy.insert(klass).values(**obj)
_ = cnx.execute(statement)
else:
new_obj = klass(**obj) # ORM object
db.session.add(new_obj)
# Stocke l'id pour les références (foreign keys):
if id_name and has_id:
if isinstance(old_id, int):
# les id int étaient utilisés pour les "entreprises"
old_id = table_name + "." + str(old_id)
id_from_scodoc7[old_id] = new_id
# tables ordonnées topologiquement pour les clés étrangères:
# g = nx.read_adjlist("misc/model-scodoc7.csv", create_using=nx.DiGraph,delimiter=";")
# L = list(reversed(list(nx.topological_sort(g))))
SCO7_TABLES_ORDONNEES = [
# (table SQL, nom de l'id scodoc7)
("notes_formations", "formation_id"),
("notes_ue", "ue_id"),
("notes_matieres", "matiere_id"),
("notes_formsemestre", "formsemestre_id"),
("notes_modules", "module_id"),
("notes_moduleimpl", "moduleimpl_id"),
(
"notes_modules_enseignants",
"modules_enseignants_id",
), # (relation) avait un id modules_enseignants_id
("partition", "partition_id"),
("identite", "etudid"),
("entreprises", "entreprise_id"),
("notes_evaluation", "evaluation_id"),
("group_descr", "group_id"),
("group_membership", "group_membership_id"), # (relation)
("notes_semset", "semset_id"),
("notes_tags", "tag_id"),
("itemsuivi", "itemsuivi_id"),
("itemsuivi_tags", "tag_id"),
("adresse", "adresse_id"),
("admissions", "adm_id"),
("absences", ""),
("scolar_news", "news_id"),
("scolog", ""),
("etud_annotations", "id"),
("billet_absence", "billet_id"),
("entreprise_correspondant", "entreprise_corresp_id"),
("entreprise_contact", "entreprise_contact_id"),
("absences_notifications", ""),
# ("notes_form_modalites", "form_modalite_id"), : déjà initialisées
("notes_appreciations", "id"),
("scolar_autorisation_inscription", "autorisation_inscription_id"),
("scolar_formsemestre_validation", "formsemestre_validation_id"),
("scolar_events", "event_id"),
("notes_notes_log", "id"),
("notes_notes", ""),
("notes_moduleimpl_inscription", "moduleimpl_inscription_id"),
("notes_formsemestre_inscription", "formsemestre_inscription_id"),
("notes_formsemestre_custommenu", "custommenu_id"),
(
"notes_formsemestre_ue_computation_expr",
"notes_formsemestre_ue_computation_expr_id",
),
("notes_formsemestre_uecoef", "formsemestre_uecoef_id"),
("notes_semset_formsemestre", ""), # (relation)
("notes_formsemestre_etapes", ""),
("notes_formsemestre_responsables", ""), # (relation)
("notes_modules_tags", ""),
("itemsuivi_tags_assoc", ""), # (relation)
("sco_prefs", "pref_id"),
]
"""
from tools.import_scodoc7_dept import *
import_scodoc7_dept( "RT" )
"""

View File

@ -17,6 +17,7 @@ def import_scodoc7_user_db(scodoc7_db="dbname=SCOUSERS"):
The resulting users are in SCO8USERS, The resulting users are in SCO8USERS,
handled via Flask/SQLAlchemy ORM. handled via Flask/SQLAlchemy ORM.
""" """
messages = []
cnx = psycopg2.connect(scodoc7_db) cnx = psycopg2.connect(scodoc7_db)
cursor = cnx.cursor(cursor_factory=psycopg2.extras.DictCursor) cursor = cnx.cursor(cursor_factory=psycopg2.extras.DictCursor)
cursor.execute("SELECT * FROM sco_users;") cursor.execute("SELECT * FROM sco_users;")
@ -46,27 +47,34 @@ def import_scodoc7_user_db(scodoc7_db="dbname=SCOUSERS"):
else: else:
roles7 = [] roles7 = []
for role_dept in roles7: for role_dept in roles7:
m = re.match(r"^-?([A-Za-z0-9]+?)([A-Z][A-Za-z0-9]*?)$", role_dept) # Cas particulier RespPeRT
m = re.match(r"^(-?RespPe)([A-Z][A-Za-z0-9]*?)$", role_dept)
if not m: if not m:
current_app.logger.warning( # Cas général: eg EnsRT
"User {}: ignoring role {}".format(u7["user_name"], role_dept) m = re.match(r"^(-?[A-Za-z0-9]+?)([A-Z][A-Za-z0-9]*?)$", role_dept)
if not m:
msg = (
f"User {u7['user_name']}: invalid role '{role_dept}' (ignoring)"
) )
current_app.logger.warning(msg)
messages.append(msg)
else: else:
role_name = m.group(1) role_name = m.group(1)
if role_name.startswith("-"): if role_name.startswith("-"):
# disabled users in ScoDoc7 # disabled users in ScoDoc7
role_name = role_name[1:] role_name = role_name[1:]
assert not u.active assert not u.active
dept = m.group(2) # silently ignore old (disabled) role
role = Role.query.filter_by(name=role_name).first()
if not role:
current_app.logger.warning(
"User {}: ignoring role {}".format(
u7["user_name"], role_dept
)
)
else: else:
u.add_role(role, dept) dept = m.group(2)
role = Role.query.filter_by(name=role_name).first()
if not role:
msg = f"User {u7['user_name']}: ignoring role '{role_dept}'"
current_app.logger.warning(msg)
messages.append(msg)
else:
u.add_role(role, dept)
db.session.add(u) db.session.add(u)
current_app.logger.info("imported user {}".format(u)) current_app.logger.info("imported user {}".format(u))
db.session.commit() db.session.commit()
return messages

View File

@ -106,13 +106,22 @@ else
fi fi
fi fi
# Migration base utilisateurs # ----- Migration base utilisateurs
echo echo
echo "Importer les utilisateurs de ScoDoc7 dans ScoDoc8 ?" echo "-------------------------------------------------------------"
echo "Importation des utilisateurs de ScoDoc7 dans ScoDoc8 "
echo "(la base SCOUSERS de ScoDoc7 sera laissée inchangée)" echo "(la base SCOUSERS de ScoDoc7 sera laissée inchangée)"
echo "(les utilisateurs ScoDoc8 existants seront laissés inchangés)" echo "(les utilisateurs ScoDoc8 existants seront laissés inchangés)"
read -r ans echo "-------------------------------------------------------------"
if [ "$(norm_ans "$ans")" != 'N' ] echo
then
(cd "$SCODOC_DIR" && flask user-db-import-scodoc7) su -c (cd "$SCODOC_DIR" && flask import-scodoc7-users)
fi
# ----- Migration bases départements
# les départements ScoDoc7 ont été déplacés dans /opt/scodoc-data/config/dept
for f in "$SCODOC_VAR_DIR"/config/depts/*.cfg
do
dept=$(basename "${f%.*}")
# Liste des bases de données de département:
psql -l | awk '{print $1;}' | grep ^SCO | grep -v SCOUSERS | grep -v SCO8USERS
done