Modified install/upgrade scripts to use Flask-Migrate (Alembic)

README.md

@@ -130,6 +130,25 @@ un utilisateur:
 **Attention:** les tests unitaires **effacent** complètement le contenu de la
 base de données (tous les départements, et les utilisateurs) avant de commencer !
 
+#### Modification du schéma de la base
+
+On utilise SQLAlchemy avec Alembic et Flask-Migrate.
+
+    flask db migrate -m "ScoDoc 9.0.4" # ajuster le message !
+    flask db upgrade
+
+Ne pas oublier de commiter les migrations (`git add migrations` ...).
+
+Mémo pour développeurs: séquence re-création d'une base:
+
+    dropdb SCODOC_DEV
+    tools/create_database.sh SCODOC_DEV # créé base SQL
+    flask db upgrade # créé les tables à partir des migrations
+    flask sco-db-init # ajoute au besoin les constantes (todo: mettre en migration 0)
+    
+    # puis imports:
+    flask import-scodoc7-users
+    flask import-scodoc7-dept STID SCOSTID
 
 # Paquet debian 11
 

app/__init__.py

@@ -25,7 +25,7 @@ from config import DevConfig
 import sco_version
 
 db = SQLAlchemy()
-migrate = Migrate()
+migrate = Migrate(compare_type=True)
 login = LoginManager()
 login.login_view = "auth.login"
 login.login_message = "Please log in to access this page."
@@ -140,7 +140,9 @@ def set_sco_dept(scodoc_dept: str):
 
 
 def user_db_init():
-    """Initialize the users database."""
+    """Initialize the users database.
+    Check that basic roles and admin user exist.
+    """
     from app.auth.models import User, Role
 
     current_app.logger.info("Init User's db")
@@ -167,8 +169,8 @@ def user_db_init():
             )
 
 
-def sco_db_init():
-    """Initialize Sco database"""
+def sco_db_insert_constants():
+    """Initialize Sco database: insert some constants (modalités, ...)."""
     from app import models
 
     current_app.logger.info("Init Sco db")
@@ -176,25 +178,25 @@ def sco_db_init():
     models.NotesFormModalite.insert_modalites()
 
 
-def initialize_scodoc_database(erase=False):
-    """Initialize the database.
+def initialize_scodoc_database(erase=False, create_all=False):
+    """Initialize the database for unit tests
     Starts from an existing database and create all necessary
     SQL tables and functions.
     If erase is True, _erase_ all database content.
     """
     from app import models
 
-    # - our specific functions and sequences, not generated by SQLAlchemy
-    models.create_database_functions()
     # - ERASE (the truncation sql function has been defined above)
     if erase:
         truncate_database()
     # - Create all tables
-    db.create_all()
+    if create_all:
+        # managed by migrations, except for TESTS
+        db.create_all()
     # - Insert initial roles and create super-admin user
     user_db_init()
     # - Insert some constant values (modalites, ...)
-    sco_db_init()
+    sco_db_insert_constants()
     # - Flush cache
     clear_scodoc_cache()
 

app/models/formations.py

@@ -87,7 +87,8 @@ class NotesModule(db.Model):
     module_id = db.synonym("id")
     titre = db.Column(db.Text())
     abbrev = db.Column(db.Text())  # nom court
-    code = db.Column(db.String(SHORT_STR_LEN), nullable=False)
+    # certains départements ont des codes infiniment longs: donc Text !
+    code = db.Column(db.Text(), nullable=False)
     heures_cours = db.Column(db.Float)
     heures_td = db.Column(db.Float)
     heures_tp = db.Column(db.Float)

app/models/raw_sql_init.py

@@ -8,8 +8,12 @@ using raw SQL
 from app import db
 
 
-def create_database_functions():
-    """Create specific SQL functions and sequences"""
+def create_database_functions():  # XXX obsolete
+    """Create specific SQL functions and sequences
+
+    XXX Obsolete: cette fonction est dans la première migration 9.0.3
+    Flask-Migrate fait maintenant (dans les versions >= 9.0.4) ce travail.
+    """
     # Important: toujours utiliser IF NOT EXISTS
     # car cette fonction peut être appelée plusieurs fois sur la même db
     db.session.execute(

migrations/README

@@ -0,0 +1 @@
+Single-database configuration for Flask.

migrations/alembic.ini

@@ -0,0 +1,50 @@
+# A generic, single database configuration.
+
+[alembic]
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic,flask_migrate
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[logger_flask_migrate]
+level = INFO
+handlers =
+qualname = flask_migrate
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

migrations/env.py

@@ -0,0 +1,91 @@
+from __future__ import with_statement
+
+import logging
+from logging.config import fileConfig
+
+from flask import current_app
+
+from alembic import context
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+logger = logging.getLogger('alembic.env')
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+config.set_main_option(
+    'sqlalchemy.url',
+    str(current_app.extensions['migrate'].db.get_engine().url).replace(
+        '%', '%%'))
+target_metadata = current_app.extensions['migrate'].db.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+
+    # this callback is used to prevent an auto-migration from being generated
+    # when there are no changes to the schema
+    # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
+    def process_revision_directives(context, revision, directives):
+        if getattr(config.cmd_opts, 'autogenerate', False):
+            script = directives[0]
+            if script.upgrade_ops.is_empty():
+                directives[:] = []
+                logger.info('No changes in schema detected.')
+
+    connectable = current_app.extensions['migrate'].db.get_engine()
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            process_revision_directives=process_revision_directives,
+            **current_app.extensions['migrate'].configure_args
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

migrations/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

migrations/versions/6b071b7947e5_scodoc_9_0_4_code_module_en_text.py

@@ -0,0 +1,34 @@
+"""ScoDoc 9.0.4: code module en Text
+
+Revision ID: 6b071b7947e5
+Revises: 993ce4a01d57
+Create Date: 2021-08-27 16:00:27.322153
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '6b071b7947e5'
+down_revision = '993ce4a01d57'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('notes_modules', 'code',
+               existing_type=sa.VARCHAR(length=32),
+               type_=sa.Text(),
+               existing_nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('notes_modules', 'code',
+               existing_type=sa.Text(),
+               type_=sa.VARCHAR(length=32),
+               existing_nullable=False)
+    # ### end Alembic commands ###

scodoc.py

@@ -66,7 +66,7 @@ def make_shell_context():
 
 
 @app.cli.command()
-def db_init():  # db-init
+def sco_db_init():  # sco-db-init
     """Initialize the database.
     Starts from an existing database and create all
     the necessary SQL tables and functions.

tests/conftest.py

@@ -23,7 +23,7 @@ def test_client():
         with apptest.app_context():
             with apptest.test_request_context():
                 # erase and reset database:
-                initialize_scodoc_database(erase=True)
+                initialize_scodoc_database(erase=True, create_all=True)
                 # Loge l'utilisateur super-admin
                 admin_user = get_super_admin()
                 login_user(admin_user)

tools/configure-scodoc9.sh

@@ -121,7 +121,7 @@ then
     echo
     echo "Création des tables et du compte admin"
     echo
-    su -c "(cd /opt/scodoc; source venv/bin/activate; flask db-init; flask user-password admin)" "$SCODOC_USER" || die "Erreur: db-init"
+    su -c "(cd /opt/scodoc; source venv/bin/activate; flask db upgrade; flask sco-db-init; flask user-password admin)" "$SCODOC_USER" || die "Erreur: sco-db-init"
     echo
     echo "base initialisée et admin créé."
     echo

tools/debian/postinst

@@ -15,7 +15,7 @@ check_create_scodoc_user
 
 # -- Répertoires /opt/scodoc donné à scodoc
 change_scodoc_file_ownership
-# --- Création au bseoin de /opt/scodoc-data
+# --- Création au besoin de /opt/scodoc-data
 set_scodoc_var_dir
 
 # ------------ LOCALES (pour compat bases ScoDoc 7 et plus anciennes)
@@ -71,11 +71,11 @@ fi
 # ------------ CREATION DU VIRTUALENV
 # donc re-créé sur le client à chaque install ou upgrade
 #echo "Creating python3 virtualenv..."
-(cd $SCODOC_DIR && python3 -m venv venv) || die "Error creating Python 3 virtualenv"
+su -c "(cd $SCODOC_DIR && python3 -m venv venv)" "$SCODOC_USER" || die "Error creating Python 3 virtualenv"
 
 # ------------ INSTALL DES PAQUETS PYTHON (3.9)
 # pip in our env, as user "scodoc"
-(cd $SCODOC_DIR && source venv/bin/activate && pip install wheel && pip install -r requirements-3.9.txt) || die "Error installing python packages"
+su -c "(cd $SCODOC_DIR && source venv/bin/activate && pip install wheel && pip install -r requirements-3.9.txt)" "$SCODOC_USER" || die "Error installing python packages"
 
 # --- NGINX
 if [ ! -L /etc/nginx/sites-enabled/scodoc9.nginx ]
@@ -89,6 +89,19 @@ fi
 # --- Ensure postgres user "scodoc" ($POSTGRES_USER) exists
 init_postgres_user
 
+# ------------  BASE DE DONNEES
+# gérées avec Flask-Migrate (Alembic/SQLAlchemy)
+# Si la base SCODOC existe, tente de la mettre à jour
+# (Ne gère pas les bases DEV et TEST)
+n=$(su -c "psql -l | grep -c -E '^[[:blank:]]*SCODOC[[:blank:]]*\|'" "$SCODOC_USER")
+if [ "$n" == 1 ]
+then
+    echo "Upgrading existing SCODOC database..."
+    # utilise les scripts dans migrations/version/
+    # pour mettre à jour notre base (en tant qu'utilisateur scodoc)
+    export SQLALCHEMY_DATABASE_URI="postgresql:///SCODOC"
+    su -c "(cd $SCODOC_DIR && source venv/bin/activate && flask db upgrade)" "$SCODOC_USER"
+fi
 
 # ------------ CONFIG SERVICE SCODOC
 echo