(en cours) Réorganisation configs prod/dev/test. Fonctions d'initialisation bases.

README.md

@@ -131,7 +131,7 @@ En tant qu'utilisateur `scodoc`:
 
 Puis initialisation de l'appli:
 
-    flask user-db-init
+    flask db-init
 
 Et saisie du mot de passe `admin`:
 

app/__init__.py

@@ -21,7 +21,7 @@ from flask_bootstrap import Bootstrap
 from flask_moment import Moment
 from flask_caching import Cache
 
-from config import Config
+from config import DevConfig
 
 from app.scodoc import notesdb as ndb
 from app.scodoc import sco_cache
@@ -47,11 +47,12 @@ def handle_sco_value_error(exc):
     return render_template("sco_value_error.html", exc=exc), 404
 
 
-def create_app(config_class=Config):
+def create_app(config_class=DevConfig):
-    print("create_app")
     app = Flask(__name__, static_url_path="/ScoDoc/static", static_folder="static")
     app.logger.setLevel(logging.DEBUG)
     app.config.from_object(config_class)
+
+    app.logger.info(f"create_app({config_class.__name__})")
     db.init_app(app)
     migrate.init_app(app, db)
     login.init_app(app)
@@ -128,3 +129,88 @@ def create_app(config_class=Config):
         app.logger.info("ScoDoc8 startup")
 
     return app
+
+
+def user_db_init():
+    """Initialize the users database."""
+    from app.auth.models import User, Role
+
+    current_app.logger.info("Init User's db")
+    # Create roles:
+    Role.insert_roles()
+    current_app.logger.info("created initial roles")
+    # Ensure that admin exists
+    admin_mail = current_app.config.get("SCODOC_ADMIN_MAIL")
+    if admin_mail:
+        admin_user_name = current_app.config["SCODOC_ADMIN_LOGIN"]
+        user = User.query.filter_by(user_name=admin_user_name).first()
+        if not user:
+            user = User(user_name=admin_user_name, email=admin_mail)
+            try:
+                db.session.add(user)
+                db.session.commit()
+            except:
+                db.session.rollback()
+                raise
+            current_app.logger.info(
+                "created initial admin user, login: {u.user_name}, email: {u.email}".format(
+                    u=user
+                )
+            )
+
+
+def sco_db_init():
+    """Initialize Sco database"""
+    from app import models
+
+    current_app.logger.info("Init Sco db")
+    # Modalités:
+    models.NotesFormModalite.insert_modalites()
+
+
+def initialize_scodoc_database(erase=False):
+    """Initialize the database.
+    Starts from an existing database and create all necessary
+    SQL tables and functions.
+    If erase is True, _erase_ all database content.
+    """
+    from app import models
+
+    # - our specific functions and sequences, not generated by SQLAlchemy
+    models.create_database_functions()
+    # - ERASE (the truncation sql function has been defined above)
+    if erase:
+        truncate_database()
+    # - Create all tables
+    db.create_all()
+    # - Insert initial roles and create super-admin user
+    user_db_init()
+    # - Insert some constant values (modalites, ...)
+    sco_db_init()
+
+
+def truncate_database():
+    """Erase content of all tables (including users !) from
+    the current database.
+    """
+    # use a stored SQL function, see createtables.sql
+    try:
+        db.session.execute("SELECT truncate_tables('scodoc');")
+        db.session.commit()
+    except:
+        db.session.rollback()
+        raise
+
+
+# admin_role = Role.query.filter_by(name="SuperAdmin").first()
+# if admin_role:
+#     admin = (
+#         User.query.join(UserRole)
+#         .filter((UserRole.user_id == User.id) & (UserRole.role_id == admin_role.id))
+#         .first()
+#     )
+# else:
+#     click.echo(
+#         "Warning: user database not initialized !\n (use: flask user-db-init)"
+#     )
+#     admin = None

app/models/__init__.py

@@ -8,7 +8,7 @@ CODE_STR_LEN = 16  # chaine pour les codes
 SHORT_STR_LEN = 32  # courtes chaine, eg acronymes
 APO_CODE_STR_LEN = 16  # nb de car max d'un code Apogée
 
-from app.models.raw_sql_init import init_scodoc_database
+from app.models.raw_sql_init import create_database_functions
 
 from app.models.absences import Absence, AbsenceNotification, BilletAbsence
 from app.models.entreprises import (

app/models/formsemestre.py

@@ -98,24 +98,30 @@ class NotesFormModalite(db.Model):
     def insert_modalites():
         """Create default modalities"""
         numero = 0
-        for (code, titre) in (
+        try:
-            (NotesFormModalite.DEFAULT_MODALITE, "Formation Initiale"),
+            for (code, titre) in (
-            ("FAP", "Apprentissage"),
+                (NotesFormModalite.DEFAULT_MODALITE, "Formation Initiale"),
-            ("FC", "Formation Continue"),
+                ("FAP", "Apprentissage"),
-            ("DEC", "Formation Décalées"),
+                ("FC", "Formation Continue"),
-            ("LIC", "Licence"),
+                ("DEC", "Formation Décalées"),
-            ("CPRO", "Contrats de Professionnalisation"),
+                ("LIC", "Licence"),
-            ("DIST", "À distance"),
+                ("CPRO", "Contrats de Professionnalisation"),
-            ("ETR", "À l'étranger"),
+                ("DIST", "À distance"),
-            ("EXT", "Extérieur"),
+                ("ETR", "À l'étranger"),
-            ("OTHER", "Autres formations"),
+                ("EXT", "Extérieur"),
-        ):
+                ("OTHER", "Autres formations"),
-            modalite = NotesFormModalite.query.filter_by(modalite=code).first()
+            ):
-            if modalite is None:
+                modalite = NotesFormModalite.query.filter_by(modalite=code).first()
-                modalite = NotesFormModalite(modalite=code, titre=titre, numero=numero)
+                if modalite is None:
-                db.session.add(modalite)
+                    modalite = NotesFormModalite(
-                numero += 1
+                        modalite=code, titre=titre, numero=numero
+                    )
+                    db.session.add(modalite)
+                    numero += 1
             db.session.commit()
+        except:
+            db.session.rollback()
+            raise
 
 
 class NotesFormsemestreUECoef(db.Model):

app/models/raw_sql_init.py

@@ -8,7 +8,7 @@ using raw SQL
 from app import db
 
 
-def init_scodoc_database():
+def create_database_functions():
     """Create specific SQL functions and sequences"""
     # Important: toujours utiliser IF NOT EXISTS
     # car cette fonction peut être appelée plusieurs fois sur la même db

config.py

@@ -37,3 +37,28 @@ class Config:
     # STATIC_URL_PATH = "/ScoDoc/static"
     # static_folder = "stat"
     # SERVER_NAME = os.environ.get("SERVER_NAME")
+
+
+class ProdConfig(Config):
+    FLASK_ENV = "production"
+    DEBUG = False
+    TESTING = False
+    SQLALCHEMY_DATABASE_URI = (
+        os.environ.get("SCODOC_DATABASE_URI") or "postgresql:///SCODOC"
+    )
+
+
+class DevConfig(Config):
+    FLASK_ENV = "development"
+    DEBUG = True
+    TESTING = True
+    SQLALCHEMY_DATABASE_URI = (
+        os.environ.get("SCODOC_DEV_DATABASE_URI") or "postgresql:///SCODOC_DEV"
+    )
+
+
+class TestConfig(DevConfig):
+    SQLALCHEMY_DATABASE_URI = (
+        os.environ.get("SCODOC_TEST_DATABASE_URI") or "postgresql:///SCODOC_TEST"
+    )
+    SERVER_NAME = "test.gr"

scodoc.py

@@ -16,7 +16,7 @@ import sys
 import click
 import flask
 from flask.cli import with_appcontext
-from app import create_app, cli, db
+from app import create_app, cli, db, initialize_scodoc_database
 
 from app.auth.models import User, Role, UserRole
 from app import models
@@ -24,9 +24,9 @@ from app import models
 from app.views import notes, scolar, absences
 import app.utils as utils
 
-from config import Config
+from config import DevConfig
 
-app = create_app()
+app = create_app(DevConfig)
 cli.register(app)
 
 
@@ -36,19 +36,6 @@ def make_shell_context():
     from app.scodoc import sco_utils as scu
     from flask_login import login_user, logout_user, current_user
 
-    admin_role = Role.query.filter_by(name="SuperAdmin").first()
-    if admin_role:
-        admin = (
-            User.query.join(UserRole)
-            .filter((UserRole.user_id == User.id) & (UserRole.role_id == admin_role.id))
-            .first()
-        )
-    else:
-        click.echo(
-            "Warning: user database not initialized !\n (use: flask user-db-init)"
-        )
-        admin = None
-
     return {
         "db": db,
         "User": User,
@@ -64,7 +51,6 @@ def make_shell_context():
         "current_user": current_user,
         "login_user": login_user,
         "logout_user": logout_user,
-        "admin": admin,
         "ctx": app.test_request_context(),
         "models": models,
     }
@@ -75,42 +61,36 @@ def make_shell_context():
 
 
 @app.cli.command()
-def user_db_init():  # user-db-init
+def db_init():  # db-init
-    """Initialize the users database."""
+    """Initialize the database.
-    click.echo("Init the db")
+    Starts from an existing database and create all
-    # Create roles:
+    the necessary SQL tables and functions.
-    Role.insert_roles()
+    """
-    click.echo("created initial roles")
+    initialize_scodoc_database()
-    # Ensure that admin exists
-    if Config.SCODOC_ADMIN_MAIL:
-        admin_user_name = Config.SCODOC_ADMIN_LOGIN
-        user = User.query.filter_by(user_name=admin_user_name).first()
-        if not user:
-            user = User(user_name=admin_user_name, email=Config.SCODOC_ADMIN_MAIL)
-            db.session.add(user)
-            db.session.commit()
-            click.echo(
-                "created initial admin user, login: {u.user_name}, email: {u.email}".format(
-                    u=user
-                )
-            )
-    # Modalités:
-    models.NotesFormModalite.insert_modalites()
 
 
 @app.cli.command()
 def user_db_clear():
-    """Erase (drop) all tables of users database !"""
+    """Erase all users and roles from the database !"""
     click.echo("Erasing the users database !")
     _clear_users_db()
 
 
 def _clear_users_db():
     """Erase (drop) all tables of users database !"""
-    click.confirm("This will erase all users.\nDo you want to continue?", abort=True)
+    click.confirm(
+        "This will erase all users and roles.\nAre you sure you want to continue?",
+        abort=True,
+    )
     db.reflect()
-    db.drop_all()
+    try:
-    db.session.commit()
+        db.session.query(UserRole).delete()
+        db.session.query(User).delete()
+        db.session.query(User).delete()
+        db.session.commit()
+    except:
+        db.session.rollback()
+        raise
 
 
 @app.cli.command()
@@ -165,6 +145,7 @@ def user_password(username, password=None):  # user-password
 @click.argument("dept")
 def sco_delete_dept(dept):  # sco-delete-dept
     "Delete existing departement"
+    raise NotImplementedError()
     if os.system('tools/delete_dept.sh -n "{}"'.format(dept)):
         sys.stderr.write("error deleting dept " + dept)
         return 1
@@ -175,6 +156,7 @@ def sco_delete_dept(dept):  # sco-delete-dept
 @click.argument("dept")
 def sco_create_dept(dept):  # sco-create-dept
     "Create new departement"
+    raise NotImplementedError()
     if os.system(f'tools/create_dept.sh -n "{dept}"'):
         sys.stderr.write(f"error creating dept {dept}\n")
         return 1

tests/conftest.py

@@ -3,40 +3,38 @@ import pytest
 from flask import g
 from flask_login import login_user, logout_user, current_user
 
-import app as myapp
+from config import TestConfig
-from app import db, create_app
+from app import db, create_app, initialize_scodoc_database
 from app import models
-from app.auth.models import User, Role, Permission
+from app.auth.models import User, Role, UserRole, Permission
 from app.scodoc import sco_bulletins_standard
 from app.scodoc import notesdb as ndb
 
 
-def truncate_database():
-    "Erase content of all tables from current dept database"
-    # use a stored SQL function, see createtables.sql
-    ndb.SimpleQuery("SELECT truncate_tables('scodoc');", {})
-
-
 @pytest.fixture()
 def test_client():
     # Setup
-    myapp.Config.TESTING = True
+    apptest = create_app(TestConfig)
-    myapp.Config.SQLALCHEMY_DATABASE_URI = "postgresql://scodoc@localhost/SCOTEST00"
-    myapp.Config.SERVER_NAME = "test.gr"
-    apptest = create_app()
     # Run tests:
     with apptest.test_client() as client:
         with apptest.app_context():
             with apptest.test_request_context():
-                models.init_scodoc_database()
+                # erase and reset database:
-                db.create_all()
+                initialize_scodoc_database(erase=True)
-                Role.insert_roles()
+                # Loge l'utilisateur super-admin
-                u = User(user_name="admin")
+                admin_role = Role.query.filter_by(name="SuperAdmin").first()
-                super_admin_role = Role.query.filter_by(name="SuperAdmin").first()
+                assert admin_role
-                u.add_role(super_admin_role, "TEST00")
+                admin_user = (
-                # u.set_password("admin")
+                    User.query.join(UserRole)
-                login_user(u)
+                    .filter(
-                # Vérifie que l'utilisateur bach existe
+                        (UserRole.user_id == User.id)
+                        & (UserRole.role_id == admin_role.id)
+                    )
+                    .first()
+                )
+                assert admin_user
+                login_user(admin_user)
+                # Vérifie que l'utilisateur "bach" existe
                 u = User.query.filter_by(user_name="bach").first()
                 if u is None:
                     u = User(user_name="bach")

tools/create_database.sh

@@ -1,9 +1,7 @@
 #!/bin/bash
 
-# Create database for a ScoDoc departement
+# Create database for ScoDoc
-# This script must be executed as postgres super user
+# This script must be executed as user "scodoc"
-#
-# $db_name is passed as an environment variable
 
 die() {
   echo 
@@ -11,16 +9,16 @@ die() {
   echo
   exit 1
 }
+[ $# = 1 ] || die "Usage $0 db_name"
+db_name="$1"
 
-source config.sh || die "config.sh not found, exiting"
+# Le répertoire de ce script:
-source utils.sh || die "config.sh not found, exiting"
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
-if [ "$db_name" == "" ]
+source "$SCRIPT_DIR"/config.sh || die "config.sh not found, exiting"
-then
+source "$SCRIPT_DIR"/utils.sh || die "config.sh not found, exiting"
-    echo "Error: env var db_name unset"
+
-    echo "(ce script ne doit pas être lancé directement !)"
+[ "$USER" = "$SCODOC_USER" ] || die "$0 must run as user $SCODOC_USER"
-    exit 1
-fi
 
 # ---
 echo 'Creating postgresql database ' "$db_name"

tools/create_dept.sh

@@ -1,91 +0,0 @@
-#!/bin/bash
-
-#
-# ScoDoc: creation initiale d'un departement
-#
-# Ce script prend en charge la creation de la base de donnees
-# et doit être lancé par l'utilisateur unix root dans le repertoire .../tools
-#                          ^^^^^^^^^^^^^^^^^^^^^
-# E. Viennet, Juin 2008
-#
-
-set -eo pipefail
-
-# Le répertoire de ce script:
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
-
-source "$SCRIPT_DIR/config.sh"
-source "$SCRIPT_DIR/utils.sh"
-
-[ "$USER" = "$SCODOC_USER" ] || die "$0 must run as user $SCODOC_USER"
-
-cd "$SCRIPT_DIR"
-
-usage() {
-  echo "$0 [-n DEPT]"
-  echo "(default to interactive mode)"
-  exit 1
-}
-[ $# = 0 ] || [ $# = 2 ] || usage
-
-if [ "$1" = "-n" ]
-then
-  interactive=0
-  if [ $# -lt 2 ]
-  then
-    usage
-  fi
-  DEPT=$2
-else
-  interactive=1
-  echo -n "Nom du departement (un mot sans ponctuation, exemple \"Info\"): "
-  read -r DEPT
-fi
-
-if [[ ! "$DEPT" =~ ^[A-Za-z0-9]+$ ]]
-then
- echo 'Nom de departement invalide !'
- exit 2
-fi
-
-export DEPT
-
-db_name=SCO$(to_upper "$DEPT")
-export db_name
-
-cfg_pathname="${SCODOC_VAR_DIR}/config/depts/$DEPT".cfg
-
-if [ -e "$cfg_pathname" ]
-then
-  echo 'Erreur: Il existe deja une configuration pour "'"$DEPT"'"'
-  exit 1
-fi
-
-# -----------------------  Create Dept database
-# (créée en tant qu'utilisateur scodoc)
-./create_database.sh
-
-# ----------------------- Initialize table database
-# POSTGRES_USER == regular unix user (scodoc)
-if [ "$interactive" = 1 ]
-then
-  ./initialize_database.sh
-else
-  ./initialize_database.sh > /dev/null 2>&1
-fi
-
-
-
-# ----------------------- Enregistre fichier config
-echo "dbname=${db_name}" > "$cfg_pathname"
-
-
-if [ "$interactive" = 1 ]
-then
-  # ----------------------- 
-  echo
-  echo " Departement $DEPT cree"
-  echo
-  echo " Attention: la base de donnees n'a pas de copies de sauvegarde"
-  echo "(XXX section à revoir en ScoDoc 8.1)" # #sco8
-fi

tools/create_users_database.sh

@@ -1,22 +0,0 @@
-#!/bin/bash
-
-# Create USERS database for ScoDoc 8
-# This script must be executed as root
-#
-# $db_name is passed as an environment variable
-
-# Le répertoire de ce script:
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
-
-source "$SCRIPT_DIR/config.sh"
-source "$SCRIPT_DIR/utils.sh"
-
-
-check_uid_root "$0"
-
-# 1--- CREATION UTILISATEUR POSTGRESQL
-init_postgres_user
-
-# 2--- CREATION BASE UTILISATEURS
-echo 'Creating postgresql database for users:' "$SCODOC_USER_DB"
-su -c "createdb -E UTF-8  -p $POSTGRES_PORT -O $SCODOC_USER $SCODOC_USER_DB" "$POSTGRES_SUPERUSER"

tools/delete_dept.sh

@@ -1,90 +0,0 @@
-#!/bin/bash
-
-#
-# ScoDoc: suppression d'un departement
-#
-# Ce script supprime la base de donnees ScoDoc d'un departement
-# *** le departement doit au prealable avoir été supprime via l'interface web ! ***
-#
-# Ne fonctionne que pour les configurations "standards" (dbname=xxx)
-#
-# Il doit être lancé par l'utilisateur unix root dans le repertoire .../tools
-#                          ^^^^^^^^^^^^^^^^^^^^^
-# E. Viennet, Sept 2008
-#
-
-# Le répertoire de ce script:
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
-
-source "$SCRIPT_DIR/config.sh"
-source "$SCRIPT_DIR/utils.sh"
-
-check_uid_root "$0"
-usage() {
-  echo "$0 [-n DEPT]"
-  echo "(default to interactive mode)"
-  exit 1
-}
-[ $# = 0 ] || [ $# = 2 ] || usage
-if [ "$1" = "-n" ]
-then
-  interactive=0
-  if [ $# -lt 2 ]
-  then
-    usage
-  fi
-  DEPT=$2
-else
-  interactive=1
-  echo
-  echo "Ce script supprime la base de donnees ScoDoc d'un departement"
-  echo
-  echo "Attention: le departement doit au prealable avoir ete supprime via l'interface web !"
-  echo "faites le AVANT d'executer ce script !!!"
-  echo
-  echo -n "Nom du departement a supprimer (un mot sans ponctuation, exemple \"Info\"): "
-  read -r DEPT
-fi
-
-if [[ ! "$DEPT" =~ ^[A-Za-z0-9]+$ ]]
-then
- echo "Nom de departement invalide !"
- exit 1
-fi
-export DEPT
-
-cfg_pathname="${SCODOC_VAR_DIR}/config/depts/$DEPT".cfg
-
-if [ -e "$cfg_pathname" ]
-then
-  # arret de ScoDoc
-  scodocctl stop
-  
-  # suppression de la base postgres
-  db_name=$(sed '/^dbname=*/!d; s///;q' < "$cfg_pathname")
-  if su -c "psql -lt" "$POSTGRES_SUPERUSER" | cut -d \| -f 1 | grep -wq "$db_name"
-  then
-    echo "Suppression de la base postgres $db_name ..."
-    su -c "dropdb $db_name" "$POSTGRES_SUPERUSER" || terminate "ne peux supprimer base de donnees $db_name"
-  else
-    echo "la base postgres $db_name n'existe pas."
-  fi
-  # suppression du fichier de config
-  /bin/rm -f "$cfg_pathname" || terminate "Ne peux supprimer $cfg_pathname"
-  # relance ScoDoc
-  if [ "$interactive" = 1 ]
-  then
-    echo -n "Demarrer le serveur ScoDoc ? (y/n) [n]"
-    read -r ans
-    if [ "$(norm_ans "$ans")" = 'Y' ]
-    then
-       scodocctl start
-    fi
-  fi
-  exit 0
-else
-  echo 'Attention: pas de configuration trouvee pour "'"$DEPT"'"'
-  echo " fichier cherché: $cfg_pathname"
-  echo " => ne fait rien."
-  exit 0
-fi