Script migration (tests en cours)

This commit is contained in:
Emmanuel Viennet 2021-08-26 14:24:25 +02:00
parent eb14607bb2
commit f63616c6b3
8 changed files with 132 additions and 47 deletions

View File

@ -1,7 +1,7 @@
# -*- mode: python -*- # -*- mode: python -*-
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
SCOVERSION = "9.0.1" SCOVERSION = "9.0.2"
SCONAME = "ScoDoc" SCONAME = "ScoDoc"

View File

@ -1,20 +1,54 @@
#!/bin/bash #!/bin/bash
# Préparation d'une release ScoDoc: # Préparation d'une release ScoDoc:
die() { # Utilise jq sur Debian 11 VM
echo; echo "Erreur: $1"; echo
exit 1 # Le répertoire de ce script: .../scodoc/tools
} SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
source "$SCRIPT_DIR/config.sh"
source "$SCRIPT_DIR/utils.sh"
# tente de trouver la version dans le source , pour vérification
SCODOC_RELEASE=$(grep SCOVERSION "$SCRIPT_DIR/../sco_version.py" | awk '{ print substr($3, 2, length($3)-2) }')
# Dernière release
GITEA_RELEASE_URL="https://scodoc.org/git/api/v1/repos/viennet/ScoDoc/releases?pre-release=true"
LAST_RELEASE_TAG=$(curl "$GITEA_RELEASE_URL" | jq ".[].tag_name" | sort | tail -1 | awk '{ print substr($1, 1, length($1)) }')
echo
echo "Version détectée dans le source: $SCODOC_RELEASE"
echo "Dernière release trouvée sur gitea: $LAST_RELEASE_TAG"
echo -n "Utiliser $LAST_RELEASE_TAG ? (y/n) [y] "
read -r ans
if [ "$(norm_ans "$ans")" != 'N' ]
then
PACKAGE_VERSION="$LAST_RELEASE_TAG"
else
echo -n "Entrer la version à générer: "
read PACKAGE_VERSION
fi
PACKAGE_NAME=scodoc9 PACKAGE_NAME=scodoc9
RELEASE_TAG=9.0.1 RELEASE_TAG="$PACKAGE_VERSION"
VERSION=9.0.1 VERSION="$PACKAGE_VERSION"
RELEASE=1 RELEASE=1
ARCH="amd64" ARCH="amd64"
FACTORY_DIR="/opt/factory" FACTORY_DIR="/opt/factory"
DEST_DIR="$PACKAGE_NAME"_"$VERSION"-"$RELEASE"_"$ARCH" DEST_DIR="$PACKAGE_NAME"_"$VERSION"-"$RELEASE"_"$ARCH"
GIT_RELEASE_URL="https://scodoc.org/git/viennet/ScoDoc/archive/${RELEASE_TAG}.tar.gz"
GIT_RELEASE_URL=https://scodoc.org/git/viennet/ScoDoc/archive/${RELEASE_TAG}.tar.gz echo "Le paquet sera $DEST_DIR.deb"
echo -n "Est-ce ok ? (y/n) [y] "
read -r ans
if [ "$(norm_ans "$ans")" != 'N' ]
then
echo "ok"
else
echo "annulation."
exit 0
fi
SCODOC_USER=scodoc SCODOC_USER=scodoc
# Safety checks # Safety checks
@ -40,6 +74,16 @@ curl -o "$archive" "$GIT_RELEASE_URL" || die "curl failure for $GIT_RELEASE_URL"
SCODOC_DIR="$optdir"/scodoc SCODOC_DIR="$optdir"/scodoc
[ -d "$SCODOC_DIR" ] || die "die Erreur: $SCODOC_DIR inexistant" [ -d "$SCODOC_DIR" ] || die "die Erreur: $SCODOC_DIR inexistant"
# Inject version (eg 9.0.2) in debian:control
sed -i.bak "s/Version: x.y.z/Version: $PACKAGE_VERSION/g" /tmp/control
# and double check
v=$(grep Version "$SCODOC_DIR/tools/debian/control" | awk '{ print $2 }')
if [ "$v" != "$PACKAGE_VERSION" ]
then
echo "error in debian control file: version mismatch (bug)"
exit 1
fi
# Puis on déplace les fichiers de config (nginx, systemd, ...) # Puis on déplace les fichiers de config (nginx, systemd, ...)
# nginx: # nginx:
mkdir -p "$slash"/etc/nginx/sites-available || die "can't mkdir nginx config" mkdir -p "$slash"/etc/nginx/sites-available || die "can't mkdir nginx config"

View File

@ -13,6 +13,7 @@ source "$SCRIPT_DIR/config.sh"
source "$SCRIPT_DIR/utils.sh" source "$SCRIPT_DIR/utils.sh"
cd /opt/scodoc || die "Error: chdir to /opt/scodoc" cd /opt/scodoc || die "Error: chdir to /opt/scodoc"
check_uid_root
# ------------ VERIF SYSTEME # ------------ VERIF SYSTEME

View File

@ -1,5 +1,5 @@
Package: scodoc9 Package: scodoc9
Version: 9.0.1 Version: x.y.z
Architecture: amd64 Architecture: amd64
Maintainer: Emmanuel Viennet <emmanuel@viennet.net> Maintainer: Emmanuel Viennet <emmanuel@viennet.net>
Description: ScoDoc 9 Description: ScoDoc 9

View File

@ -66,13 +66,13 @@ ATTRIBUTES_MAPPING = {
def setup_log(dept_acronym: str): def setup_log(dept_acronym: str):
"""log to console (stderr) and /opt/scodoc-data/log/migration78.log""" """log to console (stderr) and /opt/scodoc-data/log/migration79.log"""
log_formatter = logging.Formatter( log_formatter = logging.Formatter(
"%(asctime)s %(levelname)s (" + dept_acronym + ") %(message)s" "%(asctime)s %(levelname)s (" + dept_acronym + ") %(message)s"
) )
# Log to file: # Log to file:
logger = logging.getLogger() logger = logging.getLogger()
file_handler = logging.FileHandler("/opt/scodoc-data/log/migration78.log") file_handler = logging.FileHandler("/opt/scodoc-data/log/migration79.log")
file_handler.setFormatter(log_formatter) file_handler.setFormatter(log_formatter)
logger.addHandler(file_handler) logger.addHandler(file_handler)
# Log to stderr: # Log to stderr:

View File

@ -10,6 +10,8 @@ import psycopg2.extras
from flask import current_app from flask import current_app
from app import db from app import db
from app.auth.models import User, Role from app.auth.models import User, Role
from app.scodoc import sco_utils as scu
from tools.import_scodoc7_dept import setup_log
def import_scodoc7_user_db(scodoc7_db="dbname=SCOUSERS"): def import_scodoc7_user_db(scodoc7_db="dbname=SCOUSERS"):
@ -17,22 +19,23 @@ def import_scodoc7_user_db(scodoc7_db="dbname=SCOUSERS"):
The resulting users are in SCO8USERS, The resulting users are in SCO8USERS,
handled via Flask/SQLAlchemy ORM. handled via Flask/SQLAlchemy ORM.
""" """
setup_log("USERS")
current_app.logger.info("Importation des utilisateurs...") current_app.logger.info("Importation des utilisateurs...")
messages = [] messages = []
cnx = psycopg2.connect(scodoc7_db) cnx = psycopg2.connect(scodoc7_db)
cursor = cnx.cursor(cursor_factory=psycopg2.extras.DictCursor) cursor = cnx.cursor(cursor_factory=psycopg2.extras.DictCursor)
cursor.execute("SELECT * FROM sco_users;") cursor.execute("SELECT * FROM sco_users;")
for u7 in cursor: for u7 in cursor:
user_name = u7["user_name"].strip().replace(" ", "_") user_name = scu.sanitize_string(u7["user_name"].strip())
if user_name != u7["user_name"]: if user_name != u7["user_name"]:
current_app.logger.warning( msg = f"""Changing login '{u7["user_name"]}' to '{user_name}'"""
f"""Changin login '{u7["user_name"]}' to '{user_name}'""" current_app.logger.warning(msg)
) messages.append(msg)
if User.query.filter_by(user_name=user_name).first(): if User.query.filter_by(user_name=user_name).first():
# user with same name exists ! # user with same name exists !
current_app.logger.warning( msg = f"""User {user_name} (de {u7["user_name"]}) exists and is left unchanged"""
f"""User {user_name} (de {u7["user_name"]}) exists and is left unchanged""" current_app.logger.warning(msg)
) messages.append(msg)
else: else:
u = User( u = User(
user_name=user_name, user_name=user_name,

View File

@ -19,38 +19,57 @@ source "$SCRIPT_DIR/config.sh"
source "$SCRIPT_DIR/utils.sh" source "$SCRIPT_DIR/utils.sh"
cd "$SCODOC_DIR" || die "ScoDoc 9 non installe" cd "$SCODOC_DIR" || die "ScoDoc 9 non installe"
source "venv/bin/activate"
# Ce script doit tourner comme "root" # Ce script doit tourner comme "root"
check_uid_root "$0" check_uid_root "$0"
# En principe, l'utilisateur "scodoc" existe déjà, mais au cas où on vérifie:
check_create_scodoc_user # ScoDoc 9 doit être bien installé
[ -e .env ] && die "ScoDoc 9 mal configuré: manque .env"
# Usage # Usage
usage() {
echo "Script de migration: import des données de ScoDoc 7 dans ScoDoc 9"
echo "Ce script doit être lancé en tant que root sur le nouveau"
echo "serveur ScoDoc 9 / Debian 11."
echo
echo "Usage: $0 archive"
echo " ou $0 -m"
echo
echo " archive doit être un répertoire exporté via save_scodoc7_data.sh"
echo " sur le serveur ScoDoc 7"
echo " Avec l'option -m, effectue une migration \"en place\""
echo " avec import des données ScoDoc 7 qui étaient sur cette même"
echo " machine."
echo " Dans ce cas, le répertoire /opt/scodoc DOIT avoir été renommé"
echo " /opt/scodoc7"
echo " AVANT l'installation de ScoDoc 9."
echo
exit 1
}
if [ ! $# -eq 1 ] if [ ! $# -eq 1 ]
then then
echo "Usage: $0 directory" usage
exit 1 fi
if [ "$1" == "-h" ] || [ "$1" == "--help" ]
then
usage
fi
if [ "$1" == "-m" ]
then
echo "migration en place"
INPLACE=1
SCODOC7_HOME=/opt/scodoc7
else
echo "Migration depuis archive $1"
INPLACE=0
SCODOC7_HOME="$1" # racine de l'archive importée
fi fi
SCODOC7_HOME="$1" # racine de l'archive importée # --- 2. Propriétaire des bases de données pour import "en place"
# Bases appartenant à www-data: les attribue à "scodoc" pour le script de migration SQL
#SCODOC7_HOME="/opt/scodoc7" # qui tourne en tant que "scodoc"
#SCODOC7_DIR="${SCODOC7_HOME}/Products/ScoDoc" # Inutile si on importe via pg_restore (voir restore-scodoc7_data.sh)
#
# if [ "${SRC##*.}" == "tgz" ]
# then
# if [ -e ${SRC%%.*} ]
# then
# echo "Error: ${SRC} is an archive but ${SRC%%.*} already exists"
# exit 2
# fi
# echo "extracting archive ${SRC}..."
# tar xvfz ...
# --- 2. Propriétaire des bases de données
# Bases appartenant à www-data:
# inutile si on importe via pg_restore (voir restore-scodoc7_data.sh)
migrate_database_ownership() { migrate_database_ownership() {
SCO7_BASES=$(su -c "psql -l -t | grep www-data" "$POSTGRES_SUPERUSER" | awk -F '|' '{print $1}') SCO7_BASES=$(su -c "psql -l -t | grep www-data" "$POSTGRES_SUPERUSER" | awk -F '|' '{print $1}')
for base in $SCO7_BASES for base in $SCO7_BASES
@ -60,7 +79,12 @@ migrate_database_ownership() {
su -c "psql -c 'REASSIGN OWNED BY \"www-data\" TO scodoc;'" "$POSTGRES_SUPERUSER" su -c "psql -c 'REASSIGN OWNED BY \"www-data\" TO scodoc;'" "$POSTGRES_SUPERUSER"
} }
# --- 3. Fichiers locaux: /opt/scodoc/var => /opt/scodoc-data # --- 3. Fichiers locaux: /opt/scodoc7/var => /opt/scodoc-data
# note mémo: $SCODOC_DIR ets /opt/scodoc, et $SCODOC_VAR_DIR /opt/scodoc-data
#
# Migration en place: /opt/scodoc7/var == SCODOC7_HOME/var => /opt/scodoc-data
# Migration via archive: SCODOC7_HOME/var => /opt/scodoc-data
migrate_local_files() { migrate_local_files() {
echo "Déplacement des fichiers de configuration et des archives" echo "Déplacement des fichiers de configuration et des archives"
SCODOC_VAR_DIR_BACKUP="$SCODOC_VAR_DIR".bak SCODOC_VAR_DIR_BACKUP="$SCODOC_VAR_DIR".bak
@ -96,6 +120,11 @@ migrate_local_files() {
change_scodoc_file_ownership change_scodoc_file_ownership
if [ "$INPLACE" == 1 ]
then
migrate_database_ownership
fi
migrate_local_files migrate_local_files
set_scodoc_var_dir set_scodoc_var_dir
@ -108,17 +137,18 @@ echo
# ----- Migration base utilisateurs # ----- Migration base utilisateurs
echo echo
echo "-------------------------------------------------------------" echo "-------------------------------------------------------------"
echo "Importation des utilisateurs de ScoDoc7 dans ScoDoc 9 " echo "Importation des utilisateurs de ScoDoc 7 dans ScoDoc 9 "
echo "(la base SCOUSERS de ScoDoc7 sera laissée inchangée)" echo "(la base SCOUSERS de ScoDoc 7 sera laissée inchangée)"
echo "(les utilisateurs ScoDoc 9 existants seront laissés inchangés)" echo "(les utilisateurs ScoDoc 9 existants seront laissés inchangés)"
echo "-------------------------------------------------------------" echo "-------------------------------------------------------------"
echo echo
su -c "(cd $SCODOC_DIR && flask import-scodoc7-users)" "$SCODOC_USER" su -c "(cd $SCODOC_DIR && source venv/bin/activate && flask import-scodoc7-users)" "$SCODOC_USER"
# ----- Migration bases départements # ----- Migration bases départements
# les départements ScoDoc7 ont été déplacés dans /opt/scodoc-data/config/dept # les départements ScoDoc7 ont été déplacés dans /opt/scodoc-data/config/dept
# (ils ne sont d'ailleurs plus utilisés par ScoDoc 9)
for f in "$SCODOC_VAR_DIR"/config/depts/*.cfg for f in "$SCODOC_VAR_DIR"/config/depts/*.cfg
do do
dept=$(basename "${f%.*}") dept=$(basename "${f%.*}")
@ -127,7 +157,7 @@ do
echo "----------------------------------------------" echo "----------------------------------------------"
echo "| MIGRATION DU DEPARTEMENT $dept" echo "| MIGRATION DU DEPARTEMENT $dept"
echo "----------------------------------------------" echo "----------------------------------------------"
su -c "(cd $SCODOC_DIR && flask import-scodoc7-dept $dept $db_name)" "$SCODOC_USER" su -c "(cd $SCODOC_DIR && source venv/bin/activate && flask import-scodoc7-dept $dept $db_name)" "$SCODOC_USER"
echo "restarting postgresql server..." echo "restarting postgresql server..."
systemctl restart postgresql systemctl restart postgresql
done done

View File

@ -44,7 +44,7 @@ fi
# Safety check # Safety check
echo "Ce script recharge les donnees de votre installation ScoDoc 7" echo "Ce script recharge les donnees de votre installation ScoDoc 7"
echo "sur ce serveur pour migration vers ScoDoc 9." echo "sur ce serveur pour migration vers ScoDoc 9."
echo "Ce fichier doit avoir ete cree par le script save_scodoc_data.sh, sur une autre machine ScoDoc7." echo "Ce fichier doit avoir ete cree par le script save_scodoc_data.sh, sur une machine ScoDoc 7."
echo echo
echo -n "Voulez vous poursuivre cette operation ? (y/n) [n]" echo -n "Voulez vous poursuivre cette operation ? (y/n) [n]"
read -r ans read -r ans
@ -83,6 +83,13 @@ fi
echo "Source is $SRC" echo "Source is $SRC"
echo "Vous allez probablement voir s'afficher de nombreux messages : "
echo "pg_restore: attention : la restauration des tables avec WITH OIDS n'est plus supportée"
echo
echo "ce n'est pas grave !"
echo -n "valider pour continuer"
read -r ans
# Load postgresql dumps # Load postgresql dumps
for f in "$SRC"/SCO*.dump for f in "$SRC"/SCO*.dump
do do