From f63616c6b363651b460a447bd6005636ea05ff8c Mon Sep 17 00:00:00 2001 From: Emmanuel Viennet Date: Thu, 26 Aug 2021 14:24:25 +0200 Subject: [PATCH] Script migration (tests en cours) --- sco_version.py | 2 +- tools/build_release.sh | 58 +++++++++++++++++++--- tools/configure-scodoc9.sh | 1 + tools/debian/control | 2 +- tools/import_scodoc7_dept.py | 4 +- tools/import_scodoc7_user_db.py | 17 ++++--- tools/migrate_from_scodoc7.sh | 86 ++++++++++++++++++++++----------- tools/restore_scodoc7_data.sh | 9 +++- 8 files changed, 132 insertions(+), 47 deletions(-) diff --git a/sco_version.py b/sco_version.py index 38f59c89e4..b07e5945ac 100644 --- a/sco_version.py +++ b/sco_version.py @@ -1,7 +1,7 @@ # -*- mode: python -*- # -*- coding: utf-8 -*- -SCOVERSION = "9.0.1" +SCOVERSION = "9.0.2" SCONAME = "ScoDoc" diff --git a/tools/build_release.sh b/tools/build_release.sh index 11f621ae29..13b0564a3a 100644 --- a/tools/build_release.sh +++ b/tools/build_release.sh @@ -1,20 +1,54 @@ #!/bin/bash # Préparation d'une release ScoDoc: -die() { - echo; echo "Erreur: $1"; echo - exit 1 -} +# Utilise jq sur Debian 11 VM + +# Le répertoire de ce script: .../scodoc/tools +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" + +source "$SCRIPT_DIR/config.sh" +source "$SCRIPT_DIR/utils.sh" + +# tente de trouver la version dans le source , pour vérification +SCODOC_RELEASE=$(grep SCOVERSION "$SCRIPT_DIR/../sco_version.py" | awk '{ print substr($3, 2, length($3)-2) }') + +# Dernière release +GITEA_RELEASE_URL="https://scodoc.org/git/api/v1/repos/viennet/ScoDoc/releases?pre-release=true" + +LAST_RELEASE_TAG=$(curl "$GITEA_RELEASE_URL" | jq ".[].tag_name" | sort | tail -1 | awk '{ print substr($1, 1, length($1)) }') + +echo +echo "Version détectée dans le source: $SCODOC_RELEASE" +echo "Dernière release trouvée sur gitea: $LAST_RELEASE_TAG" +echo -n "Utiliser $LAST_RELEASE_TAG ? (y/n) [y] " +read -r ans +if [ "$(norm_ans "$ans")" != 'N' ] +then + PACKAGE_VERSION="$LAST_RELEASE_TAG" +else + echo -n "Entrer la version à générer: " + read PACKAGE_VERSION +fi PACKAGE_NAME=scodoc9 -RELEASE_TAG=9.0.1 -VERSION=9.0.1 +RELEASE_TAG="$PACKAGE_VERSION" +VERSION="$PACKAGE_VERSION" RELEASE=1 ARCH="amd64" FACTORY_DIR="/opt/factory" DEST_DIR="$PACKAGE_NAME"_"$VERSION"-"$RELEASE"_"$ARCH" +GIT_RELEASE_URL="https://scodoc.org/git/viennet/ScoDoc/archive/${RELEASE_TAG}.tar.gz" -GIT_RELEASE_URL=https://scodoc.org/git/viennet/ScoDoc/archive/${RELEASE_TAG}.tar.gz +echo "Le paquet sera $DEST_DIR.deb" +echo -n "Est-ce ok ? (y/n) [y] " +read -r ans +if [ "$(norm_ans "$ans")" != 'N' ] +then + echo "ok" +else + echo "annulation." + exit 0 +fi SCODOC_USER=scodoc # Safety checks @@ -40,6 +74,16 @@ curl -o "$archive" "$GIT_RELEASE_URL" || die "curl failure for $GIT_RELEASE_URL" SCODOC_DIR="$optdir"/scodoc [ -d "$SCODOC_DIR" ] || die "die Erreur: $SCODOC_DIR inexistant" +# Inject version (eg 9.0.2) in debian:control +sed -i.bak "s/Version: x.y.z/Version: $PACKAGE_VERSION/g" /tmp/control +# and double check +v=$(grep Version "$SCODOC_DIR/tools/debian/control" | awk '{ print $2 }') +if [ "$v" != "$PACKAGE_VERSION" ] +then + echo "error in debian control file: version mismatch (bug)" + exit 1 +fi + # Puis on déplace les fichiers de config (nginx, systemd, ...) # nginx: mkdir -p "$slash"/etc/nginx/sites-available || die "can't mkdir nginx config" diff --git a/tools/configure-scodoc9.sh b/tools/configure-scodoc9.sh index d5ebb6e550..0f2f694f4b 100755 --- a/tools/configure-scodoc9.sh +++ b/tools/configure-scodoc9.sh @@ -13,6 +13,7 @@ source "$SCRIPT_DIR/config.sh" source "$SCRIPT_DIR/utils.sh" cd /opt/scodoc || die "Error: chdir to /opt/scodoc" +check_uid_root # ------------ VERIF SYSTEME diff --git a/tools/debian/control b/tools/debian/control index f61e31b07d..05f5c30356 100644 --- a/tools/debian/control +++ b/tools/debian/control @@ -1,5 +1,5 @@ Package: scodoc9 -Version: 9.0.1 +Version: x.y.z Architecture: amd64 Maintainer: Emmanuel Viennet Description: ScoDoc 9 diff --git a/tools/import_scodoc7_dept.py b/tools/import_scodoc7_dept.py index 7f6b45855d..960376fa17 100644 --- a/tools/import_scodoc7_dept.py +++ b/tools/import_scodoc7_dept.py @@ -66,13 +66,13 @@ ATTRIBUTES_MAPPING = { def setup_log(dept_acronym: str): - """log to console (stderr) and /opt/scodoc-data/log/migration78.log""" + """log to console (stderr) and /opt/scodoc-data/log/migration79.log""" log_formatter = logging.Formatter( "%(asctime)s %(levelname)s (" + dept_acronym + ") %(message)s" ) # Log to file: logger = logging.getLogger() - file_handler = logging.FileHandler("/opt/scodoc-data/log/migration78.log") + file_handler = logging.FileHandler("/opt/scodoc-data/log/migration79.log") file_handler.setFormatter(log_formatter) logger.addHandler(file_handler) # Log to stderr: diff --git a/tools/import_scodoc7_user_db.py b/tools/import_scodoc7_user_db.py index dc8bcd7dd9..c36f13fbc3 100644 --- a/tools/import_scodoc7_user_db.py +++ b/tools/import_scodoc7_user_db.py @@ -10,6 +10,8 @@ import psycopg2.extras from flask import current_app from app import db from app.auth.models import User, Role +from app.scodoc import sco_utils as scu +from tools.import_scodoc7_dept import setup_log def import_scodoc7_user_db(scodoc7_db="dbname=SCOUSERS"): @@ -17,22 +19,23 @@ def import_scodoc7_user_db(scodoc7_db="dbname=SCOUSERS"): The resulting users are in SCO8USERS, handled via Flask/SQLAlchemy ORM. """ + setup_log("USERS") current_app.logger.info("Importation des utilisateurs...") messages = [] cnx = psycopg2.connect(scodoc7_db) cursor = cnx.cursor(cursor_factory=psycopg2.extras.DictCursor) cursor.execute("SELECT * FROM sco_users;") for u7 in cursor: - user_name = u7["user_name"].strip().replace(" ", "_") + user_name = scu.sanitize_string(u7["user_name"].strip()) if user_name != u7["user_name"]: - current_app.logger.warning( - f"""Changin login '{u7["user_name"]}' to '{user_name}'""" - ) + msg = f"""Changing login '{u7["user_name"]}' to '{user_name}'""" + current_app.logger.warning(msg) + messages.append(msg) if User.query.filter_by(user_name=user_name).first(): # user with same name exists ! - current_app.logger.warning( - f"""User {user_name} (de {u7["user_name"]}) exists and is left unchanged""" - ) + msg = f"""User {user_name} (de {u7["user_name"]}) exists and is left unchanged""" + current_app.logger.warning(msg) + messages.append(msg) else: u = User( user_name=user_name, diff --git a/tools/migrate_from_scodoc7.sh b/tools/migrate_from_scodoc7.sh index 4aa28ed5ed..e7bed49b29 100755 --- a/tools/migrate_from_scodoc7.sh +++ b/tools/migrate_from_scodoc7.sh @@ -19,38 +19,57 @@ source "$SCRIPT_DIR/config.sh" source "$SCRIPT_DIR/utils.sh" cd "$SCODOC_DIR" || die "ScoDoc 9 non installe" -source "venv/bin/activate" # Ce script doit tourner comme "root" check_uid_root "$0" -# En principe, l'utilisateur "scodoc" existe déjà, mais au cas où on vérifie: -check_create_scodoc_user + +# ScoDoc 9 doit être bien installé +[ -e .env ] && die "ScoDoc 9 mal configuré: manque .env" # Usage +usage() { + echo "Script de migration: import des données de ScoDoc 7 dans ScoDoc 9" + echo "Ce script doit être lancé en tant que root sur le nouveau" + echo "serveur ScoDoc 9 / Debian 11." + echo + echo "Usage: $0 archive" + echo " ou $0 -m" + echo + echo " archive doit être un répertoire exporté via save_scodoc7_data.sh" + echo " sur le serveur ScoDoc 7" + echo " Avec l'option -m, effectue une migration \"en place\"" + echo " avec import des données ScoDoc 7 qui étaient sur cette même" + echo " machine." + echo " Dans ce cas, le répertoire /opt/scodoc DOIT avoir été renommé" + echo " /opt/scodoc7" + echo " AVANT l'installation de ScoDoc 9." + echo + exit 1 +} if [ ! $# -eq 1 ] then - echo "Usage: $0 directory" - exit 1 + usage +fi +if [ "$1" == "-h" ] || [ "$1" == "--help" ] +then + usage +fi +if [ "$1" == "-m" ] +then + echo "migration en place" + INPLACE=1 + SCODOC7_HOME=/opt/scodoc7 +else + echo "Migration depuis archive $1" + INPLACE=0 + SCODOC7_HOME="$1" # racine de l'archive importée fi -SCODOC7_HOME="$1" # racine de l'archive importée - -#SCODOC7_HOME="/opt/scodoc7" -#SCODOC7_DIR="${SCODOC7_HOME}/Products/ScoDoc" - -# if [ "${SRC##*.}" == "tgz" ] -# then -# if [ -e ${SRC%%.*} ] -# then -# echo "Error: ${SRC} is an archive but ${SRC%%.*} already exists" -# exit 2 -# fi -# echo "extracting archive ${SRC}..." -# tar xvfz ... - -# --- 2. Propriétaire des bases de données -# Bases appartenant à www-data: -# inutile si on importe via pg_restore (voir restore-scodoc7_data.sh) +# --- 2. Propriétaire des bases de données pour import "en place" +# Bases appartenant à www-data: les attribue à "scodoc" pour le script de migration SQL +# qui tourne en tant que "scodoc" +# Inutile si on importe via pg_restore (voir restore-scodoc7_data.sh) +# migrate_database_ownership() { SCO7_BASES=$(su -c "psql -l -t | grep www-data" "$POSTGRES_SUPERUSER" | awk -F '|' '{print $1}') for base in $SCO7_BASES @@ -60,7 +79,12 @@ migrate_database_ownership() { su -c "psql -c 'REASSIGN OWNED BY \"www-data\" TO scodoc;'" "$POSTGRES_SUPERUSER" } -# --- 3. Fichiers locaux: /opt/scodoc/var => /opt/scodoc-data +# --- 3. Fichiers locaux: /opt/scodoc7/var => /opt/scodoc-data +# note mémo: $SCODOC_DIR ets /opt/scodoc, et $SCODOC_VAR_DIR /opt/scodoc-data +# +# Migration en place: /opt/scodoc7/var == SCODOC7_HOME/var => /opt/scodoc-data +# Migration via archive: SCODOC7_HOME/var => /opt/scodoc-data + migrate_local_files() { echo "Déplacement des fichiers de configuration et des archives" SCODOC_VAR_DIR_BACKUP="$SCODOC_VAR_DIR".bak @@ -96,6 +120,11 @@ migrate_local_files() { change_scodoc_file_ownership +if [ "$INPLACE" == 1 ] +then + migrate_database_ownership +fi + migrate_local_files set_scodoc_var_dir @@ -108,17 +137,18 @@ echo # ----- Migration base utilisateurs echo echo "-------------------------------------------------------------" -echo "Importation des utilisateurs de ScoDoc7 dans ScoDoc 9 " -echo "(la base SCOUSERS de ScoDoc7 sera laissée inchangée)" +echo "Importation des utilisateurs de ScoDoc 7 dans ScoDoc 9 " +echo "(la base SCOUSERS de ScoDoc 7 sera laissée inchangée)" echo "(les utilisateurs ScoDoc 9 existants seront laissés inchangés)" echo "-------------------------------------------------------------" echo -su -c "(cd $SCODOC_DIR && flask import-scodoc7-users)" "$SCODOC_USER" +su -c "(cd $SCODOC_DIR && source venv/bin/activate && flask import-scodoc7-users)" "$SCODOC_USER" # ----- Migration bases départements # les départements ScoDoc7 ont été déplacés dans /opt/scodoc-data/config/dept +# (ils ne sont d'ailleurs plus utilisés par ScoDoc 9) for f in "$SCODOC_VAR_DIR"/config/depts/*.cfg do dept=$(basename "${f%.*}") @@ -127,7 +157,7 @@ do echo "----------------------------------------------" echo "| MIGRATION DU DEPARTEMENT $dept" echo "----------------------------------------------" - su -c "(cd $SCODOC_DIR && flask import-scodoc7-dept $dept $db_name)" "$SCODOC_USER" + su -c "(cd $SCODOC_DIR && source venv/bin/activate && flask import-scodoc7-dept $dept $db_name)" "$SCODOC_USER" echo "restarting postgresql server..." systemctl restart postgresql done diff --git a/tools/restore_scodoc7_data.sh b/tools/restore_scodoc7_data.sh index a7be194471..2ab521bf42 100755 --- a/tools/restore_scodoc7_data.sh +++ b/tools/restore_scodoc7_data.sh @@ -44,7 +44,7 @@ fi # Safety check echo "Ce script recharge les donnees de votre installation ScoDoc 7" echo "sur ce serveur pour migration vers ScoDoc 9." -echo "Ce fichier doit avoir ete cree par le script save_scodoc_data.sh, sur une autre machine ScoDoc7." +echo "Ce fichier doit avoir ete cree par le script save_scodoc_data.sh, sur une machine ScoDoc 7." echo echo -n "Voulez vous poursuivre cette operation ? (y/n) [n]" read -r ans @@ -83,6 +83,13 @@ fi echo "Source is $SRC" +echo "Vous allez probablement voir s'afficher de nombreux messages : " +echo "pg_restore: attention : la restauration des tables avec WITH OIDS n'est plus supportée" +echo +echo "ce n'est pas grave !" +echo -n "valider pour continuer" +read -r ans + # Load postgresql dumps for f in "$SRC"/SCO*.dump do