From d99b9aeb59d6c9e60e2733ab3c520a2d165650d7 Mon Sep 17 00:00:00 2001
From: Emmanuel Viennet <emmanuel.viennet@gmail.com>
Date: Wed, 18 Aug 2021 19:36:30 +0200
Subject: [PATCH] Fix script import

---
 tools/import_scodoc7_dept.py    |  16 ++--
 tools/import_scodoc7_user_db.py |   1 +
 tools/migrate_from_scodoc7.sh   |   2 -
 tools/restore_scodoc_data.sh    | 141 --------------------------------
 tools/save_scodoc_data.sh       |  70 ----------------
 5 files changed, 10 insertions(+), 220 deletions(-)
 delete mode 100755 tools/restore_scodoc_data.sh
 delete mode 100755 tools/save_scodoc_data.sh

diff --git a/tools/import_scodoc7_dept.py b/tools/import_scodoc7_dept.py
index 353e302b..be183716 100644
--- a/tools/import_scodoc7_dept.py
+++ b/tools/import_scodoc7_dept.py
@@ -116,9 +116,9 @@ def import_scodoc7_dept(dept_id: str, dept_db_uri=None):
         klass = get_class_for_table(table)
         t1 = time.time()
         n = convert_table(dept, cursor, id_from_scodoc7, klass, id_name, default_user)
-        logging.info(f"     inserted {n} objects in {time.time()-t1}s.")
+        logging.info(f"     inserted {n} objects in {time.time()-t1:3.2f}s.")
 
-    logging.info(f"All table imported in {time.time()-t0}s")
+    logging.info(f"All table imported in {time.time()-t0:3.2f}s")
     logging.info(f"clearing app caches...")
     clear_scodoc_cache()
     logging.info(f"Done.")
@@ -236,7 +236,7 @@ def convert_table(
     if cnx:
         cnx.close()
 
-    db.session.commit()  # écrit la table en une fois
+    db.session.commit()  # écrit la table
     if has_id:
         update_table_sequence(table_name)
     return len(objects)
@@ -339,9 +339,9 @@ def convert_object(
         statement = sqlalchemy.insert(klass).values(**obj)
         _ = cnx.execute(statement)
     else:
-        # new_obj = klass(**obj)  # ORM object
-        # db.session.add(new_obj)
-        insert_object(cnx, table_name, obj)
+        new_obj = klass(**obj)  # ORM object
+        db.session.add(new_obj)
+        # insert_object(cnx, table_name, obj)
 
     # Stocke l'id pour les références (foreign keys):
     if id_name and has_id:
@@ -352,7 +352,9 @@ def convert_object(
 
 
 def insert_object(cnx, table_name: str, vals: dict) -> str:
-    """insert tuple in db"""
+    """insert tuple in db
+    version manuelle => ne semble pas plus rapide
+    """
     cols = list(vals.keys())
     colnames = ",".join(cols)
     fmt = ",".join(["%%(%s)s" % col for col in cols])
diff --git a/tools/import_scodoc7_user_db.py b/tools/import_scodoc7_user_db.py
index 39827e87..aa2d1a0c 100644
--- a/tools/import_scodoc7_user_db.py
+++ b/tools/import_scodoc7_user_db.py
@@ -17,6 +17,7 @@ def import_scodoc7_user_db(scodoc7_db="dbname=SCOUSERS"):
     The resulting users are in SCO8USERS,
     handled via Flask/SQLAlchemy ORM.
     """
+    current_app.logger.info("Importation des utilisateurs...")
     messages = []
     cnx = psycopg2.connect(scodoc7_db)
     cursor = cnx.cursor(cursor_factory=psycopg2.extras.DictCursor)
diff --git a/tools/migrate_from_scodoc7.sh b/tools/migrate_from_scodoc7.sh
index abc5eeff..09663c38 100755
--- a/tools/migrate_from_scodoc7.sh
+++ b/tools/migrate_from_scodoc7.sh
@@ -104,8 +104,6 @@ echo "Les fichiers locaux de ScoDoc: configuration, photos, procès-verbaux..."
 echo "sont maintenant stockées dans $SCODOC_VAR_DIR"
 echo
 
-echo "XXX STOPPING XXX"
-exit 0
 
 # ----- Migration base utilisateurs
 echo
diff --git a/tools/restore_scodoc_data.sh b/tools/restore_scodoc_data.sh
deleted file mode 100755
index 90a88f4f..00000000
--- a/tools/restore_scodoc_data.sh
+++ /dev/null
@@ -1,141 +0,0 @@
-#!/bin/bash
-
-#
-# ScoDoc:  restore data (saved by save_scodoc_data) into current install
-# 
-#  Utile pour migrer ScoDoc d'un serveur a un autre
-#  A executer en tant que root sur le nouveau serveur
-#
-# E. Viennet, Sept 2011, Nov 2013, Mar 2017, Aug 2020, Jul 2021
-#
-
-# Le répertoire de ce script:
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
-
-source "$SCRIPT_DIR/config.sh"
-source "$SCRIPT_DIR/utils.sh"
-
-check_uid_root "$0"
-
-# Safety check
-echo "Ce script va remplacer les donnees de votre installation ScoDoc par celles"
-echo "enregistrees dans le fichier fourni."
-echo "Ce fichier doit avoir ete cree par le script save_scodoc_data.sh, sur une autre machine."
-echo 
-echo "Attention: TOUTES LES DONNEES DE CE SERVEUR SERONT REMPLACEES !"
-echo "Notamment, tous les utilisateurs et departements existants seront effaces !"
-echo
-echo "TOUTES LES BASES POSTGRESQL SERONT EFFACEES !!!"
-echo 
-echo -n "Voulez vous poursuivre cette operation ? (y/n) [n]"
-read -r ans
-if [ ! "$(norm_ans "$ans")" = 'Y' ]
-then
-   echo "Annulation"
-   exit 1
-fi
-
-# Usage
-if [ ! $# -eq 1 ]
-then
-  echo "Usage: $0 directory_or_archive"
-  exit 1
-fi
-
-SRC=$1
-
-if [ "${SRC:0:1}" != "/" ]
-then
-  echo "Usage: $0 directory_or_archive"
-  echo "Erreur: utiliser un chemin absolu (commencant par /)"
-  exit 1
-fi
-
-# Source directory
-if [ "${SRC##*.}" = 'tgz' ]
-then
-  echo "Opening tgz archive..."
-  tmp=$(mktemp -d)
-  chmod a+rx "$tmp"
-  cd "$tmp" || terminate "directory error"
-  tar xfz "$SRC" 
-  SRC=$(ls -1d "$tmp"/*)
-  IS_TMP=1
-  # If source is a tgz, can use mv
-  COPY="mv"
-else
-  IS_TMP=0
-  # If source is a directory, does not modify its content
-  COPY="cp -rp"
-fi
-
-echo "Source is $SRC"
-echo "Stopping ScoDoc..."
-scodocctl stop
-
-# Erase all postgres databases and load data
-chmod a+rx "$SRC"
-chmod a+r "$SRC"/scodoc.dump.txt
-PG_DUMPFILE="$SRC/scodoc.dump.txt"
-
-su -c "$SCODOC_DIR/tools/psql_restore_databases.sh $PG_DUMPFILE" postgres
-
-# 
-echo Copying data files...
-
-rm -rf "${SCODOC_DIR:?}/var"
-$COPY "$SRC/var" "$SCODOC_DIR"
-
-if [ ! -e "${SCODOC_VAR_DIR:?}/config/" ]
-then
-  mkdir "${SCODOC_VAR_DIR:?}/config/"
-  chown "$SCODOC_USER"."$SCODOC_GROUP" "${SCODOC_VAR_DIR:?}/config/"
-  chmod 755 "${SCODOC_VAR_DIR:?}/config/"
-fi
-
-rm -rf "${SCODOC_DIR:?}/config/depts" 
-if [ -e "${SRC:?}/depts" ]
-then
-  # legacy depts => move them to var
-  $COPY "$SRC/depts" "${SCODOC_VAR_DIR}/config/"
-fi
-
-rm -rf  "$SCODOC_DIR/static/photos" 
-if [ -e "$SRC/photos" ]
-then
-  # legacy photos (in <src>/static/photos) => move them to var
-  $COPY "$SRC/photos" "${SCODOC_VAR_DIR}/"
-fi 
-
-rm -rf "${SCODOC_DIR:?}/logos"
-$COPY "$SRC/logos" "$SCODOC_DIR/"
-
-mv "$SCODOC_DIR/config/scodoc_config.py"  "$SCODOC_DIR/config/scodoc_config.py.$(date +%Y%m%d-%H%M%S)" 
-$COPY "$SRC/scodoc_config.py" "$SCODOC_DIR/config/"
-# Verifie le codage de ce fichier:
-if [ -z "$(file $SCODOC_DIR/config/scodoc_config.py | grep -i UTF-8)" ] 
-then
-   mv "$SCODOC_DIR/config/scodoc_config.py" "$SCODOC_DIR/config/scodoc_config.py.orig"
-   iconv -f iso8859-15 -t utf-8 "$SCODOC_DIR/config/scodoc_config.py.orig" > "$SCODOC_DIR/config/scodoc_config.py"
-fi
-
-rm -rf "${SCODOC_DIR:?}/log"
-$COPY "$SRC/log" "$SCODOC_DIR/"
-
-# Fix file ownership and access rights
-chown -R "$SCODOC_USER"."$SCODOC_GROUP" "${SCODOC_DIR}"
-chmod -R 755 "$SCODOC_DIR"
-
-# Remove tmp directory
-if [ "$IS_TMP" = "1" ]
-then
-  rm -rf "${tmp}"
-fi
-
-# Mise a jour BD ScoDoc
-cd ${SCODOC_DIR:?}/config || terminate "no config directory"
-./upgrade.sh
-
-#
-echo
-scodocctl start
\ No newline at end of file
diff --git a/tools/save_scodoc_data.sh b/tools/save_scodoc_data.sh
deleted file mode 100755
index 9dc7e6c1..00000000
--- a/tools/save_scodoc_data.sh
+++ /dev/null
@@ -1,70 +0,0 @@
-#!/bin/bash
-
-#
-# ScoDoc: save all user data (database, configs, images, archives...) in separate directory
-# 
-#  Utile pour migrer ScoDoc d'un serveur a un autre
-#  Executer en tant que root sur le serveur d'origine
-#
-# E. Viennet, Sept 2011, Aug 2020, Jul 2021
-#
-# Le répertoire de ce script:
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
-
-source "$SCRIPT_DIR/config.sh"
-source "$SCRIPT_DIR/utils.sh"
-
-
-check_uid_root "$0"
-
-# Destination directory
-if [ ! $# -eq 1 ]
-then
-  echo "Usage: $0 destination_directory"
-  exit 1
-fi
-DEST=$1
-# remove trailing slashs if needed:
-shopt -s extglob
-DEST="${DEST%%+(/)}"
-
-if [ ! -e "$DEST" ]
-then
-  echo Creating directory "$DEST"
-  mkdir "$DEST"
-else
-  echo "Error: Directory " "$DEST"  " exists"
-  echo "remove it or specify another destination !"
-  exit 2
-fi
-
-echo "Stopping ScoDoc..."
-scodocctl stop
-
-# Dump all postgres databases
-echo "Dumping SQL database..."
-chown postgres "$DEST"
-su -c "pg_dumpall > \"$DEST\"/scodoc.dump.txt" postgres
-if [ ! "$?" -eq 0 ] 
-then
-  printf "Error dumping postgresql database\nPlease check that SQL server is running\nAborting.\n"
-  exit 1
-fi
-chown root "$DEST"
-
-# ScoDoc archives, configuration, photos, etc.
-echo "Copying var/ ..." 
-cp -rp "$SCODOC_DIR/var" "$DEST"
-
-
-echo "Copying server logs..."
-cp -rp "$SCODOC_DIR/log" "$DEST"
-
-
-# --- Archive all files in a tarball to ease transfer
-echo
-echo "Archiving backup files in a $DEST.tgz..."
-base=$(basename "$DEST")
-(cd "$DEST"/.. || terminate "directory error"; tar cfz "$DEST".tgz "$base")
-
-echo "Done (you can copy " "$DEST"".tgz to destination machine)."