Reset serial init values, and configure logging

This commit is contained in:
Emmanuel Viennet 2021-08-15 09:09:49 +02:00
parent aeb03ada68
commit 3049545796

View File

@ -2,6 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import inspect import inspect
import logging
import pdb import pdb
import psycopg2 import psycopg2
@ -15,6 +16,70 @@ import app
from app import models from app import models
from app.scodoc import notesdb as ndb from app.scodoc import notesdb as ndb
# Attributs modifiés entre les bases ScoDoc 7 et 8+:
# (None indique que l'attribut est supprimé, "nouveau_nom" qu'il change de nom)
ATTRIBUTES_MAPPING = {
"admissions": {
"debouche": None,
},
"adresse": {
"entreprise_id": None,
},
"etud_annotations": {
"zope_authenticated_user": "author",
"zope_remote_addr": None,
},
"identite": {
"foto": None,
},
"notes_formsemestre": {
"etape_apo2": None, # => suppressed
"etape_apo3": None,
"etape_apo4": None,
# préférences, plus dans formsemestre:
# (inutilisés depuis ScoDoc 6 environ)
"bul_show_decision": None,
"bul_show_uevalid": None,
"nomgroupetd": None,
"nomgroupetp": None,
"nomgroupeta": None,
"gestion_absence": None,
"bul_show_codemodules": None,
"bul_show_rangs": None,
"bul_show_ue_rangs": None,
"bul_show_mod_rangs": None,
},
"partition": {
"compute_ranks": None,
},
"notes_appreciations": {
"zope_authenticated_user": "author",
"zope_remote_addr": None,
},
"scolog": {
"remote_addr": None,
"remote_host": None,
},
}
def setup_log(dept_id: str):
"""log to console (stderr) and /opt/scodoc-data/log/migration78.log"""
log_formatter = logging.Formatter(
"%(asctime)s %(levelname)s (" + dept_id + ") %(message)s"
)
# Log to file:
logger = logging.getLogger()
file_handler = logging.FileHandler("/opt/scodoc-data/log/migration78.log")
file_handler.setFormatter(log_formatter)
logger.addHandler(file_handler)
# Log to stderr:
console_handler = logging.StreamHandler() # stderr
console_handler.setFormatter(log_formatter)
logger.addHandler(console_handler)
# Caution:
logger.setLevel(logging.DEBUG)
def import_scodoc7_dept(dept_id: str, dept_db_uri=None): def import_scodoc7_dept(dept_id: str, dept_db_uri=None):
"""Importe un département ScoDoc7 dans ScoDoc >= 8.1 """Importe un département ScoDoc7 dans ScoDoc >= 8.1
@ -30,6 +95,7 @@ def import_scodoc7_dept(dept_id: str, dept_db_uri=None):
raise ValueError(f"le département {dept_id} existe déjà !") raise ValueError(f"le département {dept_id} existe déjà !")
if dept_db_uri is None: if dept_db_uri is None:
dept_db_uri = f"postgresql:///SCO{dept_id}" dept_db_uri = f"postgresql:///SCO{dept_id}"
setup_log(dept_id)
current_app.logger.info(f"connecting to database {dept_db_uri}") current_app.logger.info(f"connecting to database {dept_db_uri}")
cnx = psycopg2.connect(dept_db_uri) cnx = psycopg2.connect(dept_db_uri)
cursor = cnx.cursor(cursor_factory=ndb.ScoDocCursor) cursor = cnx.cursor(cursor_factory=ndb.ScoDocCursor)
@ -81,8 +147,8 @@ def get_table_max_id(klass):
if not id in sqlalchemy.inspect(klass).columns.keys(): if not id in sqlalchemy.inspect(klass).columns.keys():
return -1 return -1
sql_table = str(klass.description) sql_table = str(klass.description)
con = db.engine.connect() cnx = db.engine.connect()
r = con.execute("SELECT max(id) FROM " + sql_table) r = cnx.execute("SELECT max(id) FROM " + sql_table)
r.fetchone() r.fetchone()
if r: if r:
return r[0] return r[0]
@ -90,6 +156,19 @@ def get_table_max_id(klass):
return 0 return 0
def update_table_sequence(table_name):
"""After filling the table, we need to update the serial
so that the next insertions will use new ids
"""
with db.engine.connect() as cnx:
cnx.execute(
f"""SELECT
setval('{table_name}_id_seq',
(SELECT MAX(id) FROM {table_name}))
"""
)
def convert_table( def convert_table(
dept, cursor, id_from_scodoc7: dict, klass=None, id_name=None, default_user=None dept, cursor, id_from_scodoc7: dict, klass=None, id_name=None, default_user=None
): ):
@ -143,53 +222,11 @@ def convert_table(
cnx.close() cnx.close()
db.session.commit() # écrit la table en une fois db.session.commit() # écrit la table en une fois
if has_id:
update_table_sequence(table_name)
return len(objects) return len(objects)
ATTRIBUTES_MAPPING = {
"admissions": {
"debouche": None,
},
"adresse": {
"entreprise_id": None,
},
"etud_annotations": {
"zope_authenticated_user": "author",
"zope_remote_addr": None,
},
"identite": {
"foto": None,
},
"notes_formsemestre": {
"etape_apo2": None, # => suppressed
"etape_apo3": None,
"etape_apo4": None,
# préférences, plus dans formsemestre:
"bul_show_decision": None,
"bul_show_uevalid": None,
"nomgroupetd": None,
"nomgroupetp": None,
"nomgroupeta": None,
"gestion_absence": None,
"bul_show_codemodules": None,
"bul_show_rangs": None,
"bul_show_ue_rangs": None,
"bul_show_mod_rangs": None,
},
"partition": {
"compute_ranks": None,
},
"notes_appreciations": {
"zope_authenticated_user": "author",
"zope_remote_addr": None,
},
"scolog": {
"remote_addr": None,
"remote_host": None,
},
}
def convert_object( def convert_object(
new_id, new_id,
dept, dept,
@ -316,7 +353,7 @@ SCO7_TABLES_ORDONNEES = [
("entreprises", "entreprise_id"), ("entreprises", "entreprise_id"),
("notes_evaluation", "evaluation_id"), ("notes_evaluation", "evaluation_id"),
("group_descr", "group_id"), ("group_descr", "group_id"),
("group_membership", "group_membership_id"), # (relation) ("group_membership", "group_membership_id"), # (relation, qui avait un id)
("notes_semset", "semset_id"), ("notes_semset", "semset_id"),
("notes_tags", "tag_id"), ("notes_tags", "tag_id"),
("itemsuivi", "itemsuivi_id"), ("itemsuivi", "itemsuivi_id"),