From 30495457969a176ee9969c19411e1afbd89ac7a0 Mon Sep 17 00:00:00 2001 From: Emmanuel Viennet Date: Sun, 15 Aug 2021 09:09:49 +0200 Subject: [PATCH] Reset serial init values, and configure logging --- tools/import_scodoc7_dept.py | 131 ++++++++++++++++++++++------------- 1 file changed, 84 insertions(+), 47 deletions(-) diff --git a/tools/import_scodoc7_dept.py b/tools/import_scodoc7_dept.py index dfd6d6033..23f2b9fd3 100644 --- a/tools/import_scodoc7_dept.py +++ b/tools/import_scodoc7_dept.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import inspect +import logging import pdb import psycopg2 @@ -15,6 +16,70 @@ import app from app import models from app.scodoc import notesdb as ndb +# Attributs modifiés entre les bases ScoDoc 7 et 8+: +# (None indique que l'attribut est supprimé, "nouveau_nom" qu'il change de nom) +ATTRIBUTES_MAPPING = { + "admissions": { + "debouche": None, + }, + "adresse": { + "entreprise_id": None, + }, + "etud_annotations": { + "zope_authenticated_user": "author", + "zope_remote_addr": None, + }, + "identite": { + "foto": None, + }, + "notes_formsemestre": { + "etape_apo2": None, # => suppressed + "etape_apo3": None, + "etape_apo4": None, + # préférences, plus dans formsemestre: + # (inutilisés depuis ScoDoc 6 environ) + "bul_show_decision": None, + "bul_show_uevalid": None, + "nomgroupetd": None, + "nomgroupetp": None, + "nomgroupeta": None, + "gestion_absence": None, + "bul_show_codemodules": None, + "bul_show_rangs": None, + "bul_show_ue_rangs": None, + "bul_show_mod_rangs": None, + }, + "partition": { + "compute_ranks": None, + }, + "notes_appreciations": { + "zope_authenticated_user": "author", + "zope_remote_addr": None, + }, + "scolog": { + "remote_addr": None, + "remote_host": None, + }, +} + + +def setup_log(dept_id: str): + """log to console (stderr) and /opt/scodoc-data/log/migration78.log""" + log_formatter = logging.Formatter( + "%(asctime)s %(levelname)s (" + dept_id + ") %(message)s" + ) + # Log to file: + logger = logging.getLogger() + file_handler = logging.FileHandler("/opt/scodoc-data/log/migration78.log") + file_handler.setFormatter(log_formatter) + logger.addHandler(file_handler) + # Log to stderr: + console_handler = logging.StreamHandler() # stderr + console_handler.setFormatter(log_formatter) + logger.addHandler(console_handler) + # Caution: + logger.setLevel(logging.DEBUG) + def import_scodoc7_dept(dept_id: str, dept_db_uri=None): """Importe un département ScoDoc7 dans ScoDoc >= 8.1 @@ -30,6 +95,7 @@ def import_scodoc7_dept(dept_id: str, dept_db_uri=None): raise ValueError(f"le département {dept_id} existe déjà !") if dept_db_uri is None: dept_db_uri = f"postgresql:///SCO{dept_id}" + setup_log(dept_id) current_app.logger.info(f"connecting to database {dept_db_uri}") cnx = psycopg2.connect(dept_db_uri) cursor = cnx.cursor(cursor_factory=ndb.ScoDocCursor) @@ -81,8 +147,8 @@ def get_table_max_id(klass): if not id in sqlalchemy.inspect(klass).columns.keys(): return -1 sql_table = str(klass.description) - con = db.engine.connect() - r = con.execute("SELECT max(id) FROM " + sql_table) + cnx = db.engine.connect() + r = cnx.execute("SELECT max(id) FROM " + sql_table) r.fetchone() if r: return r[0] @@ -90,6 +156,19 @@ def get_table_max_id(klass): return 0 +def update_table_sequence(table_name): + """After filling the table, we need to update the serial + so that the next insertions will use new ids + """ + with db.engine.connect() as cnx: + cnx.execute( + f"""SELECT + setval('{table_name}_id_seq', + (SELECT MAX(id) FROM {table_name})) + """ + ) + + def convert_table( dept, cursor, id_from_scodoc7: dict, klass=None, id_name=None, default_user=None ): @@ -143,53 +222,11 @@ def convert_table( cnx.close() db.session.commit() # écrit la table en une fois + if has_id: + update_table_sequence(table_name) return len(objects) -ATTRIBUTES_MAPPING = { - "admissions": { - "debouche": None, - }, - "adresse": { - "entreprise_id": None, - }, - "etud_annotations": { - "zope_authenticated_user": "author", - "zope_remote_addr": None, - }, - "identite": { - "foto": None, - }, - "notes_formsemestre": { - "etape_apo2": None, # => suppressed - "etape_apo3": None, - "etape_apo4": None, - # préférences, plus dans formsemestre: - "bul_show_decision": None, - "bul_show_uevalid": None, - "nomgroupetd": None, - "nomgroupetp": None, - "nomgroupeta": None, - "gestion_absence": None, - "bul_show_codemodules": None, - "bul_show_rangs": None, - "bul_show_ue_rangs": None, - "bul_show_mod_rangs": None, - }, - "partition": { - "compute_ranks": None, - }, - "notes_appreciations": { - "zope_authenticated_user": "author", - "zope_remote_addr": None, - }, - "scolog": { - "remote_addr": None, - "remote_host": None, - }, -} - - def convert_object( new_id, dept, @@ -316,7 +353,7 @@ SCO7_TABLES_ORDONNEES = [ ("entreprises", "entreprise_id"), ("notes_evaluation", "evaluation_id"), ("group_descr", "group_id"), - ("group_membership", "group_membership_id"), # (relation) + ("group_membership", "group_membership_id"), # (relation, qui avait un id) ("notes_semset", "semset_id"), ("notes_tags", "tag_id"), ("itemsuivi", "itemsuivi_id"),