Modified install/upgrade scripts to use Flask-Migrate (Alembic)

This commit is contained in:
Emmanuel Viennet 2021-08-27 17:03:47 +02:00
parent 137a486329
commit 6090672089
14 changed files with 1457 additions and 19 deletions

View File

@ -130,6 +130,25 @@ un utilisateur:
**Attention:** les tests unitaires **effacent** complètement le contenu de la
base de données (tous les départements, et les utilisateurs) avant de commencer !
#### Modification du schéma de la base
On utilise SQLAlchemy avec Alembic et Flask-Migrate.
flask db migrate -m "ScoDoc 9.0.4" # ajuster le message !
flask db upgrade
Ne pas oublier de commiter les migrations (`git add migrations` ...).
Mémo pour développeurs: séquence re-création d'une base:
dropdb SCODOC_DEV
tools/create_database.sh SCODOC_DEV # créé base SQL
flask db upgrade # créé les tables à partir des migrations
flask sco-db-init # ajoute au besoin les constantes (todo: mettre en migration 0)
# puis imports:
flask import-scodoc7-users
flask import-scodoc7-dept STID SCOSTID
# Paquet debian 11

View File

@ -25,7 +25,7 @@ from config import DevConfig
import sco_version
db = SQLAlchemy()
migrate = Migrate()
migrate = Migrate(compare_type=True)
login = LoginManager()
login.login_view = "auth.login"
login.login_message = "Please log in to access this page."
@ -140,7 +140,9 @@ def set_sco_dept(scodoc_dept: str):
def user_db_init():
"""Initialize the users database."""
"""Initialize the users database.
Check that basic roles and admin user exist.
"""
from app.auth.models import User, Role
current_app.logger.info("Init User's db")
@ -167,8 +169,8 @@ def user_db_init():
)
def sco_db_init():
"""Initialize Sco database"""
def sco_db_insert_constants():
"""Initialize Sco database: insert some constants (modalités, ...)."""
from app import models
current_app.logger.info("Init Sco db")
@ -176,25 +178,25 @@ def sco_db_init():
models.NotesFormModalite.insert_modalites()
def initialize_scodoc_database(erase=False):
"""Initialize the database.
def initialize_scodoc_database(erase=False, create_all=False):
"""Initialize the database for unit tests
Starts from an existing database and create all necessary
SQL tables and functions.
If erase is True, _erase_ all database content.
"""
from app import models
# - our specific functions and sequences, not generated by SQLAlchemy
models.create_database_functions()
# - ERASE (the truncation sql function has been defined above)
if erase:
truncate_database()
# - Create all tables
db.create_all()
if create_all:
# managed by migrations, except for TESTS
db.create_all()
# - Insert initial roles and create super-admin user
user_db_init()
# - Insert some constant values (modalites, ...)
sco_db_init()
sco_db_insert_constants()
# - Flush cache
clear_scodoc_cache()

View File

@ -87,7 +87,8 @@ class NotesModule(db.Model):
module_id = db.synonym("id")
titre = db.Column(db.Text())
abbrev = db.Column(db.Text()) # nom court
code = db.Column(db.String(SHORT_STR_LEN), nullable=False)
# certains départements ont des codes infiniment longs: donc Text !
code = db.Column(db.Text(), nullable=False)
heures_cours = db.Column(db.Float)
heures_td = db.Column(db.Float)
heures_tp = db.Column(db.Float)

View File

@ -8,8 +8,12 @@ using raw SQL
from app import db
def create_database_functions():
"""Create specific SQL functions and sequences"""
def create_database_functions(): # XXX obsolete
"""Create specific SQL functions and sequences
XXX Obsolete: cette fonction est dans la première migration 9.0.3
Flask-Migrate fait maintenant (dans les versions >= 9.0.4) ce travail.
"""
# Important: toujours utiliser IF NOT EXISTS
# car cette fonction peut être appelée plusieurs fois sur la même db
db.session.execute(

1
migrations/README Executable file
View File

@ -0,0 +1 @@
Single-database configuration for Flask.

50
migrations/alembic.ini Normal file
View File

@ -0,0 +1,50 @@
# A generic, single database configuration.
[alembic]
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic,flask_migrate
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[logger_flask_migrate]
level = INFO
handlers =
qualname = flask_migrate
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

91
migrations/env.py Executable file
View File

@ -0,0 +1,91 @@
from __future__ import with_statement
import logging
from logging.config import fileConfig
from flask import current_app
from alembic import context
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
logger = logging.getLogger('alembic.env')
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
config.set_main_option(
'sqlalchemy.url',
str(current_app.extensions['migrate'].db.get_engine().url).replace(
'%', '%%'))
target_metadata = current_app.extensions['migrate'].db.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url, target_metadata=target_metadata, literal_binds=True
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
connectable = current_app.extensions['migrate'].db.get_engine()
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
**current_app.extensions['migrate'].configure_args
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

24
migrations/script.py.mako Executable file
View File

@ -0,0 +1,24 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

View File

@ -0,0 +1,34 @@
"""ScoDoc 9.0.4: code module en Text
Revision ID: 6b071b7947e5
Revises: 993ce4a01d57
Create Date: 2021-08-27 16:00:27.322153
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '6b071b7947e5'
down_revision = '993ce4a01d57'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('notes_modules', 'code',
existing_type=sa.VARCHAR(length=32),
type_=sa.Text(),
existing_nullable=False)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.alter_column('notes_modules', 'code',
existing_type=sa.Text(),
type_=sa.VARCHAR(length=32),
existing_nullable=False)
# ### end Alembic commands ###

File diff suppressed because it is too large Load Diff

View File

@ -66,7 +66,7 @@ def make_shell_context():
@app.cli.command()
def db_init(): # db-init
def sco_db_init(): # sco-db-init
"""Initialize the database.
Starts from an existing database and create all
the necessary SQL tables and functions.

View File

@ -23,7 +23,7 @@ def test_client():
with apptest.app_context():
with apptest.test_request_context():
# erase and reset database:
initialize_scodoc_database(erase=True)
initialize_scodoc_database(erase=True, create_all=True)
# Loge l'utilisateur super-admin
admin_user = get_super_admin()
login_user(admin_user)

View File

@ -121,7 +121,7 @@ then
echo
echo "Création des tables et du compte admin"
echo
su -c "(cd /opt/scodoc; source venv/bin/activate; flask db-init; flask user-password admin)" "$SCODOC_USER" || die "Erreur: db-init"
su -c "(cd /opt/scodoc; source venv/bin/activate; flask db upgrade; flask sco-db-init; flask user-password admin)" "$SCODOC_USER" || die "Erreur: sco-db-init"
echo
echo "base initialisée et admin créé."
echo

View File

@ -15,7 +15,7 @@ check_create_scodoc_user
# -- Répertoires /opt/scodoc donné à scodoc
change_scodoc_file_ownership
# --- Création au bseoin de /opt/scodoc-data
# --- Création au besoin de /opt/scodoc-data
set_scodoc_var_dir
# ------------ LOCALES (pour compat bases ScoDoc 7 et plus anciennes)
@ -71,11 +71,11 @@ fi
# ------------ CREATION DU VIRTUALENV
# donc re-créé sur le client à chaque install ou upgrade
#echo "Creating python3 virtualenv..."
(cd $SCODOC_DIR && python3 -m venv venv) || die "Error creating Python 3 virtualenv"
su -c "(cd $SCODOC_DIR && python3 -m venv venv)" "$SCODOC_USER" || die "Error creating Python 3 virtualenv"
# ------------ INSTALL DES PAQUETS PYTHON (3.9)
# pip in our env, as user "scodoc"
(cd $SCODOC_DIR && source venv/bin/activate && pip install wheel && pip install -r requirements-3.9.txt) || die "Error installing python packages"
su -c "(cd $SCODOC_DIR && source venv/bin/activate && pip install wheel && pip install -r requirements-3.9.txt)" "$SCODOC_USER" || die "Error installing python packages"
# --- NGINX
if [ ! -L /etc/nginx/sites-enabled/scodoc9.nginx ]
@ -89,6 +89,19 @@ fi
# --- Ensure postgres user "scodoc" ($POSTGRES_USER) exists
init_postgres_user
# ------------ BASE DE DONNEES
# gérées avec Flask-Migrate (Alembic/SQLAlchemy)
# Si la base SCODOC existe, tente de la mettre à jour
# (Ne gère pas les bases DEV et TEST)
n=$(su -c "psql -l | grep -c -E '^[[:blank:]]*SCODOC[[:blank:]]*\|'" "$SCODOC_USER")
if [ "$n" == 1 ]
then
echo "Upgrading existing SCODOC database..."
# utilise les scripts dans migrations/version/
# pour mettre à jour notre base (en tant qu'utilisateur scodoc)
export SQLALCHEMY_DATABASE_URI="postgresql:///SCODOC"
su -c "(cd $SCODOC_DIR && source venv/bin/activate && flask db upgrade)" "$SCODOC_USER"
fi
# ------------ CONFIG SERVICE SCODOC
echo