Fix script import

This commit is contained in:
Emmanuel Viennet 2021-08-18 19:36:30 +02:00
parent 42a439eaac
commit d99b9aeb59
5 changed files with 10 additions and 220 deletions

View File

@ -116,9 +116,9 @@ def import_scodoc7_dept(dept_id: str, dept_db_uri=None):
klass = get_class_for_table(table)
t1 = time.time()
n = convert_table(dept, cursor, id_from_scodoc7, klass, id_name, default_user)
logging.info(f" inserted {n} objects in {time.time()-t1}s.")
logging.info(f" inserted {n} objects in {time.time()-t1:3.2f}s.")
logging.info(f"All table imported in {time.time()-t0}s")
logging.info(f"All table imported in {time.time()-t0:3.2f}s")
logging.info(f"clearing app caches...")
clear_scodoc_cache()
logging.info(f"Done.")
@ -236,7 +236,7 @@ def convert_table(
if cnx:
cnx.close()
db.session.commit() # écrit la table en une fois
db.session.commit() # écrit la table
if has_id:
update_table_sequence(table_name)
return len(objects)
@ -339,9 +339,9 @@ def convert_object(
statement = sqlalchemy.insert(klass).values(**obj)
_ = cnx.execute(statement)
else:
# new_obj = klass(**obj) # ORM object
# db.session.add(new_obj)
insert_object(cnx, table_name, obj)
new_obj = klass(**obj) # ORM object
db.session.add(new_obj)
# insert_object(cnx, table_name, obj)
# Stocke l'id pour les références (foreign keys):
if id_name and has_id:
@ -352,7 +352,9 @@ def convert_object(
def insert_object(cnx, table_name: str, vals: dict) -> str:
"""insert tuple in db"""
"""insert tuple in db
version manuelle => ne semble pas plus rapide
"""
cols = list(vals.keys())
colnames = ",".join(cols)
fmt = ",".join(["%%(%s)s" % col for col in cols])

View File

@ -17,6 +17,7 @@ def import_scodoc7_user_db(scodoc7_db="dbname=SCOUSERS"):
The resulting users are in SCO8USERS,
handled via Flask/SQLAlchemy ORM.
"""
current_app.logger.info("Importation des utilisateurs...")
messages = []
cnx = psycopg2.connect(scodoc7_db)
cursor = cnx.cursor(cursor_factory=psycopg2.extras.DictCursor)

View File

@ -104,8 +104,6 @@ echo "Les fichiers locaux de ScoDoc: configuration, photos, procès-verbaux..."
echo "sont maintenant stockées dans $SCODOC_VAR_DIR"
echo
echo "XXX STOPPING XXX"
exit 0
# ----- Migration base utilisateurs
echo

View File

@ -1,141 +0,0 @@
#!/bin/bash
#
# ScoDoc: restore data (saved by save_scodoc_data) into current install
#
# Utile pour migrer ScoDoc d'un serveur a un autre
# A executer en tant que root sur le nouveau serveur
#
# E. Viennet, Sept 2011, Nov 2013, Mar 2017, Aug 2020, Jul 2021
#
# Le répertoire de ce script:
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
source "$SCRIPT_DIR/config.sh"
source "$SCRIPT_DIR/utils.sh"
check_uid_root "$0"
# Safety check
echo "Ce script va remplacer les donnees de votre installation ScoDoc par celles"
echo "enregistrees dans le fichier fourni."
echo "Ce fichier doit avoir ete cree par le script save_scodoc_data.sh, sur une autre machine."
echo
echo "Attention: TOUTES LES DONNEES DE CE SERVEUR SERONT REMPLACEES !"
echo "Notamment, tous les utilisateurs et departements existants seront effaces !"
echo
echo "TOUTES LES BASES POSTGRESQL SERONT EFFACEES !!!"
echo
echo -n "Voulez vous poursuivre cette operation ? (y/n) [n]"
read -r ans
if [ ! "$(norm_ans "$ans")" = 'Y' ]
then
echo "Annulation"
exit 1
fi
# Usage
if [ ! $# -eq 1 ]
then
echo "Usage: $0 directory_or_archive"
exit 1
fi
SRC=$1
if [ "${SRC:0:1}" != "/" ]
then
echo "Usage: $0 directory_or_archive"
echo "Erreur: utiliser un chemin absolu (commencant par /)"
exit 1
fi
# Source directory
if [ "${SRC##*.}" = 'tgz' ]
then
echo "Opening tgz archive..."
tmp=$(mktemp -d)
chmod a+rx "$tmp"
cd "$tmp" || terminate "directory error"
tar xfz "$SRC"
SRC=$(ls -1d "$tmp"/*)
IS_TMP=1
# If source is a tgz, can use mv
COPY="mv"
else
IS_TMP=0
# If source is a directory, does not modify its content
COPY="cp -rp"
fi
echo "Source is $SRC"
echo "Stopping ScoDoc..."
scodocctl stop
# Erase all postgres databases and load data
chmod a+rx "$SRC"
chmod a+r "$SRC"/scodoc.dump.txt
PG_DUMPFILE="$SRC/scodoc.dump.txt"
su -c "$SCODOC_DIR/tools/psql_restore_databases.sh $PG_DUMPFILE" postgres
#
echo Copying data files...
rm -rf "${SCODOC_DIR:?}/var"
$COPY "$SRC/var" "$SCODOC_DIR"
if [ ! -e "${SCODOC_VAR_DIR:?}/config/" ]
then
mkdir "${SCODOC_VAR_DIR:?}/config/"
chown "$SCODOC_USER"."$SCODOC_GROUP" "${SCODOC_VAR_DIR:?}/config/"
chmod 755 "${SCODOC_VAR_DIR:?}/config/"
fi
rm -rf "${SCODOC_DIR:?}/config/depts"
if [ -e "${SRC:?}/depts" ]
then
# legacy depts => move them to var
$COPY "$SRC/depts" "${SCODOC_VAR_DIR}/config/"
fi
rm -rf "$SCODOC_DIR/static/photos"
if [ -e "$SRC/photos" ]
then
# legacy photos (in <src>/static/photos) => move them to var
$COPY "$SRC/photos" "${SCODOC_VAR_DIR}/"
fi
rm -rf "${SCODOC_DIR:?}/logos"
$COPY "$SRC/logos" "$SCODOC_DIR/"
mv "$SCODOC_DIR/config/scodoc_config.py" "$SCODOC_DIR/config/scodoc_config.py.$(date +%Y%m%d-%H%M%S)"
$COPY "$SRC/scodoc_config.py" "$SCODOC_DIR/config/"
# Verifie le codage de ce fichier:
if [ -z "$(file $SCODOC_DIR/config/scodoc_config.py | grep -i UTF-8)" ]
then
mv "$SCODOC_DIR/config/scodoc_config.py" "$SCODOC_DIR/config/scodoc_config.py.orig"
iconv -f iso8859-15 -t utf-8 "$SCODOC_DIR/config/scodoc_config.py.orig" > "$SCODOC_DIR/config/scodoc_config.py"
fi
rm -rf "${SCODOC_DIR:?}/log"
$COPY "$SRC/log" "$SCODOC_DIR/"
# Fix file ownership and access rights
chown -R "$SCODOC_USER"."$SCODOC_GROUP" "${SCODOC_DIR}"
chmod -R 755 "$SCODOC_DIR"
# Remove tmp directory
if [ "$IS_TMP" = "1" ]
then
rm -rf "${tmp}"
fi
# Mise a jour BD ScoDoc
cd ${SCODOC_DIR:?}/config || terminate "no config directory"
./upgrade.sh
#
echo
scodocctl start

View File

@ -1,70 +0,0 @@
#!/bin/bash
#
# ScoDoc: save all user data (database, configs, images, archives...) in separate directory
#
# Utile pour migrer ScoDoc d'un serveur a un autre
# Executer en tant que root sur le serveur d'origine
#
# E. Viennet, Sept 2011, Aug 2020, Jul 2021
#
# Le répertoire de ce script:
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
source "$SCRIPT_DIR/config.sh"
source "$SCRIPT_DIR/utils.sh"
check_uid_root "$0"
# Destination directory
if [ ! $# -eq 1 ]
then
echo "Usage: $0 destination_directory"
exit 1
fi
DEST=$1
# remove trailing slashs if needed:
shopt -s extglob
DEST="${DEST%%+(/)}"
if [ ! -e "$DEST" ]
then
echo Creating directory "$DEST"
mkdir "$DEST"
else
echo "Error: Directory " "$DEST" " exists"
echo "remove it or specify another destination !"
exit 2
fi
echo "Stopping ScoDoc..."
scodocctl stop
# Dump all postgres databases
echo "Dumping SQL database..."
chown postgres "$DEST"
su -c "pg_dumpall > \"$DEST\"/scodoc.dump.txt" postgres
if [ ! "$?" -eq 0 ]
then
printf "Error dumping postgresql database\nPlease check that SQL server is running\nAborting.\n"
exit 1
fi
chown root "$DEST"
# ScoDoc archives, configuration, photos, etc.
echo "Copying var/ ..."
cp -rp "$SCODOC_DIR/var" "$DEST"
echo "Copying server logs..."
cp -rp "$SCODOC_DIR/log" "$DEST"
# --- Archive all files in a tarball to ease transfer
echo
echo "Archiving backup files in a $DEST.tgz..."
base=$(basename "$DEST")
(cd "$DEST"/.. || terminate "directory error"; tar cfz "$DEST".tgz "$base")
echo "Done (you can copy " "$DEST"".tgz to destination machine)."