Reset serial init values, and configure logging

tools/import_scodoc7_dept.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 
 import inspect
+import logging
 import pdb
 
 import psycopg2
@@ -15,6 +16,70 @@ import app
 from app import models
 from app.scodoc import notesdb as ndb
 
+# Attributs modifiés entre les bases ScoDoc 7 et 8+:
+# (None indique que l'attribut est supprimé, "nouveau_nom" qu'il change de nom)
+ATTRIBUTES_MAPPING = {
+    "admissions": {
+        "debouche": None,
+    },
+    "adresse": {
+        "entreprise_id": None,
+    },
+    "etud_annotations": {
+        "zope_authenticated_user": "author",
+        "zope_remote_addr": None,
+    },
+    "identite": {
+        "foto": None,
+    },
+    "notes_formsemestre": {
+        "etape_apo2": None,  # => suppressed
+        "etape_apo3": None,
+        "etape_apo4": None,
+        # préférences, plus dans formsemestre:
+        # (inutilisés depuis ScoDoc 6 environ)
+        "bul_show_decision": None,
+        "bul_show_uevalid": None,
+        "nomgroupetd": None,
+        "nomgroupetp": None,
+        "nomgroupeta": None,
+        "gestion_absence": None,
+        "bul_show_codemodules": None,
+        "bul_show_rangs": None,
+        "bul_show_ue_rangs": None,
+        "bul_show_mod_rangs": None,
+    },
+    "partition": {
+        "compute_ranks": None,
+    },
+    "notes_appreciations": {
+        "zope_authenticated_user": "author",
+        "zope_remote_addr": None,
+    },
+    "scolog": {
+        "remote_addr": None,
+        "remote_host": None,
+    },
+}
+
+
+def setup_log(dept_id: str):
+    """log to console (stderr) and /opt/scodoc-data/log/migration78.log"""
+    log_formatter = logging.Formatter(
+        "%(asctime)s %(levelname)s (" + dept_id + ")  %(message)s"
+    )
+    # Log to file:
+    logger = logging.getLogger()
+    file_handler = logging.FileHandler("/opt/scodoc-data/log/migration78.log")
+    file_handler.setFormatter(log_formatter)
+    logger.addHandler(file_handler)
+    # Log to stderr:
+    console_handler = logging.StreamHandler()  # stderr
+    console_handler.setFormatter(log_formatter)
+    logger.addHandler(console_handler)
+    # Caution:
+    logger.setLevel(logging.DEBUG)
+
 
 def import_scodoc7_dept(dept_id: str, dept_db_uri=None):
     """Importe un département ScoDoc7 dans ScoDoc >= 8.1
@@ -30,6 +95,7 @@ def import_scodoc7_dept(dept_id: str, dept_db_uri=None):
         raise ValueError(f"le département {dept_id} existe déjà !")
     if dept_db_uri is None:
         dept_db_uri = f"postgresql:///SCO{dept_id}"
+    setup_log(dept_id)
     current_app.logger.info(f"connecting to database {dept_db_uri}")
     cnx = psycopg2.connect(dept_db_uri)
     cursor = cnx.cursor(cursor_factory=ndb.ScoDocCursor)
@@ -81,8 +147,8 @@ def get_table_max_id(klass):
     if not id in sqlalchemy.inspect(klass).columns.keys():
         return -1
     sql_table = str(klass.description)
-    con = db.engine.connect()
+    cnx = db.engine.connect()
-    r = con.execute("SELECT max(id) FROM " + sql_table)
+    r = cnx.execute("SELECT max(id) FROM " + sql_table)
     r.fetchone()
     if r:
         return r[0]
@@ -90,6 +156,19 @@ def get_table_max_id(klass):
         return 0
 
 
+def update_table_sequence(table_name):
+    """After filling the table, we need to update the serial
+    so that the next insertions will use new ids
+    """
+    with db.engine.connect() as cnx:
+        cnx.execute(
+            f"""SELECT 
+        setval('{table_name}_id_seq', 
+            (SELECT MAX(id) FROM {table_name}))
+        """
+        )
+
+
 def convert_table(
     dept, cursor, id_from_scodoc7: dict, klass=None, id_name=None, default_user=None
 ):
@@ -143,53 +222,11 @@ def convert_table(
         cnx.close()
 
     db.session.commit()  # écrit la table en une fois
+    if has_id:
+        update_table_sequence(table_name)
     return len(objects)
 
 
-ATTRIBUTES_MAPPING = {
-    "admissions": {
-        "debouche": None,
-    },
-    "adresse": {
-        "entreprise_id": None,
-    },
-    "etud_annotations": {
-        "zope_authenticated_user": "author",
-        "zope_remote_addr": None,
-    },
-    "identite": {
-        "foto": None,
-    },
-    "notes_formsemestre": {
-        "etape_apo2": None,  # => suppressed
-        "etape_apo3": None,
-        "etape_apo4": None,
-        # préférences, plus dans formsemestre:
-        "bul_show_decision": None,
-        "bul_show_uevalid": None,
-        "nomgroupetd": None,
-        "nomgroupetp": None,
-        "nomgroupeta": None,
-        "gestion_absence": None,
-        "bul_show_codemodules": None,
-        "bul_show_rangs": None,
-        "bul_show_ue_rangs": None,
-        "bul_show_mod_rangs": None,
-    },
-    "partition": {
-        "compute_ranks": None,
-    },
-    "notes_appreciations": {
-        "zope_authenticated_user": "author",
-        "zope_remote_addr": None,
-    },
-    "scolog": {
-        "remote_addr": None,
-        "remote_host": None,
-    },
-}
-
-
 def convert_object(
     new_id,
     dept,
@@ -316,7 +353,7 @@ SCO7_TABLES_ORDONNEES = [
     ("entreprises", "entreprise_id"),
     ("notes_evaluation", "evaluation_id"),
     ("group_descr", "group_id"),
-    ("group_membership", "group_membership_id"),  # (relation)
+    ("group_membership", "group_membership_id"),  # (relation, qui avait un id)
     ("notes_semset", "semset_id"),
     ("notes_tags", "tag_id"),
     ("itemsuivi", "itemsuivi_id"),