installmgr/app/routes.py

import json, datetime, fcntl, glob, os, re, socket, subprocess, time, requests

from flask import jsonify, request, abort
from flask import Blueprint
from app import email
from dotenv import load_dotenv

load_dotenv()

bp = Blueprint("routes", __name__)

# --------------------------------------------------------------
DIR = "/opt/installmgr/"
REPOSIT_DIR = "/opt/installmgr/incoming_dumps"
MAX_REPOSIT_SIZE = 300 * 20 * 1024  # kB (here, max 300 dumps of 20MB)

ALERT_MAIL_FROM = "root@scodoc.org"
ALERT_MAIL_TO = "emmanuel.viennet@gmail.com"

LOG_FILENAME = os.path.join(DIR, "upload-dump-errors.log")
UPLOAD_LOG_FILENAME = os.path.join(DIR, "upload-dump-log.json")
DEBUG = False  # if false, don't publish error messages

# Les paquets publiés:
DEBIAN_PACKAGES_EXP = "/srv/packages/pool/main/s/scodoc9/scodoc9_*.deb"
RELEASE_LOG_FILE = "/home/viennet/scodoc-releases.log"

GITEA_URL = "https://scodoc.org/git"
GITEA_REPO = "ScoDoc/ScoDoc"
GITEA_LABEL_ID = None


@bp.route("/scodoc-installmgr/check_version/<client_version>")
def check_version(client_version: str):
    """check version vs release
    return json
    {
        "status": "ok",
        "client_version" : "9.1.81", // la version du client
        "client_version_date" : int // timestamp version
        "last_version" : "9.1.82" // derniere version dispo
        "last_version_date": int // timestamp derniere version
    }
    Le client devrait se plaindre de manque de mise à jour si
    (client_version != last_version) et (now - last_version_date) > 1 jour

    Si la version du client n'est pas trouvée dans RELEASE_LOG_FILE
    renvoie client_version et client_version_date nulls.
    """
    try:
        with open(RELEASE_LOG_FILE) as f:
            data = [l.strip().split() for l in f]
    except FileNotFoundError:
        return jsonify({"status": "error 1"})
    if len(data) < 1:
        return jsonify({"status": "error 2"})
    release_date = {k: v for v, k in data}  # version : date
    last_version = data[-1][1]
    last_version_date_str = data[-1][0]
    try:
        dt = datetime.datetime.strptime(last_version_date_str, "%Y-%m-%dT%H:%M:%S%z")
    except ValueError:
        return jsonify({"status": "error 3"})
    last_version_date = dt.timestamp()
    #
    client_version_date_str = release_date.get(client_version, None)
    if client_version_date_str is None:
        client_version = None  # not found ? (client altered version ?)
        client_version_date = None
    else:
        try:
            dt = datetime.datetime.strptime(
                client_version_date_str, "%Y-%m-%dT%H:%M:%S%z"
            )
            client_version_date = dt.timestamp()
        except ValueError:
            return jsonify({"status": "error 4"})
    return jsonify(
        {
            "status": "ok",
            "client_version": client_version,
            "client_version_date": client_version_date,
            "last_version": last_version,
            "last_version_date": last_version_date,
        }
    )


@bp.route("/scodoc-installmgr/last_stable_version")
def last_stable_version():
    """version du dernier paquet ScoDoc 9 publié.
    => json
    """
    # LAST_RELEASE_TAG=$(curl "$GITEA_RELEASE_URL" | jq ".[].tag_name" | tr -d -c "0-9.\n" | sort --version-sort | tail -1)
    debs = glob.glob(DEBIAN_PACKAGES_EXP)
    version_tuples = []  # (9,1,81)
    for filename in debs:
        m = re.match(r".*scodoc9_9\.([0-9]{1,2})\.([0-9]{1,3})-1_amd64.deb", filename)
        if m:
            version_tuples.append((9, int(m.group(1)), int(m.group(2)), filename))

    if len(version_tuples) == 0:
        return "?.?.?"
    version_tuples.sort()
    last_package_version = version_tuples[-1][:-1]
    last_package_filename = version_tuples[-1][-1]
    package_mtime = os.path.getmtime(last_package_filename)
    package_version_string = ".".join([str(x) for x in last_package_version])
    return jsonify(
        {
            "publication_time": package_mtime,  # float, time
            "version": package_version_string,
        }
    )


@bp.route("/scodoc-installmgr/report", methods=["POST"])
def report():
    """Création d'un ticket Gitea depuis ScoDoc
    => json
    """
    log = open(LOG_FILENAME, "a")
    log.write("report\n")
    now = datetime.datetime.now()
    fulltime = now.isoformat()
    remote_addr = request.environ.get("HTTP_X_REAL_IP", request.remote_addr)
    log.write(fulltime + " request from " + remote_addr + "\n")
    log.flush()

    request_data = request.get_json(silent=True)

    if request_data is None:
        log.write("json proccessing error\n")
        log.close()
        return (
            jsonify(
                {
                    "message": "Une erreur est survenue lors du traitement de la requête. Veuillez réessayer."
                }
            ),
            400,
        )

    ticket = request_data.get("ticket", "")
    user = request_data.get("user", "")
    dump = request_data.get("dump", "")
    scodoc = request_data.get("scodoc", "")

    if (
        not ticket
        or not user
        or not dump
        or not scodoc
        or not ticket.get("title")
        or not ticket.get("message")
    ):
        log.write("missing json fields\n")
        log.close()
        return (
            jsonify(
                {
                    "message": "Une erreur est survenue lors du traitement de la requête (données requises manquantes). Veuillez réessayer."
                }
            ),
            400,
        )

    meta = "Informations complémentaires :"
    meta += "\n- Version de ScoDoc : " + scodoc.get("version", "inconnue")
    meta += "\n- Dump : " + (
        ("inclus (id : " + dump.get("id", "inconnu") + ")")
        if dump.get("included", False)
        else "non inclus"
    )
    meta += "\n- Établissement : " + ticket.get("etab") if ticket.get("etab") else ""
    meta += "\n- Département : " + ticket.get("dept") if ticket.get("dept") else ""

    ticket_body = ticket.get("message") + "\n\n---\n\n" + meta

    response = requests.post(
        GITEA_URL + "/api/v1/repos/" + GITEA_REPO + "/issues",
        json={
            "title": ticket.get("title"),
            "body": ticket_body,
            "labels": [GITEA_LABEL_ID],
        },
        headers={"Authorization": "token " + os.getenv("GITEA_TOKEN")},
    )

    if response.status_code != 201:
        log.write("gitea error\n")
        try:
            log.write("sending notification to {}\n".format(ALERT_MAIL_TO))
            email.send_email(
                "[report] Gitea error !",
                ALERT_MAIL_FROM,
                [ALERT_MAIL_TO],
                "Error "
                + response.status_code
                + " while creating the gitea ticket :\n"
                + response.text
                + "\n\nTicket info : "
                + ticket.get("title")
                + "\n"
                + ticket_body
                + "\n- Utilisateur : "
                + user.get("name", "Nom inconnu")
                + " <"
                + user.get("email", "Adresse email inconnue")
                + ">",
            )
        except:
            log.write("exception while sending email !\n")
        log.close()
        return (
            jsonify(
                {
                    "message": "Une erreur est survenue lors de la création du ticket. Veuillez réessayer."
                }
            ),
            500,
        )

    try:
        log.write("sending notification to {}\n".format(ALERT_MAIL_TO))
        email.send_email(
            "[report] Ticket # "
            + response.json()["id"]
            + " créé avec succès ! : "
            + ticket.get("title"),
            ALERT_MAIL_FROM,
            [ALERT_MAIL_TO],
            "Nouveau ticket utilisateur :\n"
            + meta
            + "\n- Lien du ticket : "
            + response.json()["url"]
            + "\n- Utilisateur : "
            + user.get("name", "Nom inconnu")
            + " <"
            + user.get("email", "Adresse email inconnue")
            + ">",
        )
    except:
        log.write("exception while sending email !\n")
        log.close()

    return (
        jsonify(
            {
                "message": "Votre demande a été enregistrée. Vous pouvez suivre son avancement sur <a target='_blank' href='"
                + response.json()["html_url"]
                + "'>"
                + response.json()["html_url"]
                + "</a>. Vous êtes susceptible d'être contacté(e) par email à l'adresse liée à votre compte ScoDoc si des informations supplémentatires sont nécéssaires."
            }
        ),
        201,
    )


@bp.route("/scodoc-installmgr/upload-dump", methods=["POST"])
def upload_scodoc9():
    """Réception d'un fichier de dump"""
    log = open(LOG_FILENAME, "a")
    log.write("haallo\n")
    now = datetime.datetime.now()
    fulltime = now.isoformat()
    # client addr:
    remote_addr = request.environ.get("HTTP_X_REAL_IP", request.remote_addr)
    log.write(f"{fulltime} request from {remote_addr}\n")
    log.flush()
    # Avec seulement alphanum et tiret:
    clean_deptname = re.sub(r"[^A-Za-z-]", "", request.form["dept_name"])
    if not clean_deptname:
        return (
            jsonify(
                {
                    "message": 'Erreur: champ dept_name manquant.\n Merci de contacter <a href="mailto:'
                    + ALERT_MAIL_TO
                    + '">'
                    + ALERT_MAIL_TO
                    + "</a></p>"
                }
            ),
            400,
        )
    try:
        remote_host = socket.gethostbyaddr(remote_addr)[0]
    except:
        log.write("reverse DNS lookup failed for {}".format(remote_addr))
        remote_host = ""

    the_file = request.files["file"]
    filename = the_file.filename
    data = the_file.read()
    D = {
        "dept_name": request.form["dept_name"],
        "serial": request.form["serial"],
        "sco_user": request.form["sco_user"],
        "sent_by": request.form["sent_by"],
        "sco_version": request.form.get("sco_version", ""),  # release
        "sco_subversion": request.form.get("sco_subversion", ""),
        "dump_filename": fulltime + "_" + clean_deptname + ".gz",
        "dump_size": len(data),
        "message": request.form.get("message", ""),
        "request_url": request.form.get("request_url", ""),
        "remote_addr": remote_addr,
        "remote_host": remote_host,
    }

    log.write("received data ({} bytes)\n".format(D["dump_size"]))
    json_descr = json.dumps(D, sort_keys=True, indent=4)
    # --- Check disk space

    cur_size = int(
        subprocess.check_output(["du", "-skx", REPOSIT_DIR])
        .decode("utf-8")
        .split("\t")[0]
    )
    if (cur_size + len(data) / 1024) > MAX_REPOSIT_SIZE:
        # out of space !
        log.write(
            "Out of space: cur_size={}kB, limit={}\n".format(cur_size, MAX_REPOSIT_SIZE)
        )
        # Send alert
        try:
            email.send_email(
                "[upload-dump] Out of space !",
                ALERT_MAIL_FROM,
                [ALERT_MAIL_TO],
                "Out space !\nNew upload was canceled:\n" + json_descr,
            )
        except:
            log.write("exception while sending email !\n")
        log.close()
        return (
            jsonify(
                {
                    "message": 'Erreur: espace de stockage insuffisant.\n Merci de contacter <a href="mailto:'
                    + ALERT_MAIL_TO
                    + '">'
                    + ALERT_MAIL_TO
                    + "</a></p>"
                }
            ),
            507,
        )
    else:
        log.write("writing dump to {}\n".format(D["dump_filename"]))
        # dump:
        f = open(os.path.join(REPOSIT_DIR, D["dump_filename"]), "wb")
        f.write(data)
        f.close()
        uplog = open(UPLOAD_LOG_FILENAME, "a")
        uplog.write(json_descr)
        uplog.write("\n,\n")  # separator
        uplog.close()

        # Send notification
        try:
            log.write("sending notification to {}\n".format(ALERT_MAIL_TO))
            email.send_email(
                f"[upload-dump] new dump {D['dept_name']} from {D['remote_addr']} ({D['remote_host']})",
                ALERT_MAIL_FROM,
                [ALERT_MAIL_TO],
                "New upload:\n" + json_descr,
            )
        except:
            log.write("exception while sending email !\n")

    log.close()
    return jsonify(
        {
            "message": "Données envoyées",
            "dump_id": fulltime + "_" + clean_deptname,
        }
    )


@bp.route("/scodoc-installmgr/scodoc9")
def scodoc9():
    """
    Réception d'un fichier de dump uploadé
    """
    log = open(LOG_FILENAME, "a")
    log.write("hello\n")


# Lock for counter
class Lock:
    def acquire(self):
        self.f = open("lock", "w")
        fcntl.flock(self.f.fileno(), fcntl.LOCK_EX)

    def release(self):
        self.f.close()


def increment():
    L = Lock()
    L.acquire()
    try:
        try:
            val = int(open(DIR + "/counter").read())
        except:
            val = 0
        val += 1
        open("counter", "w").write("%d" % val)
    finally:
        L.release()
    return val


@bp.route("/scodoc-installmgr/version", methods=["GET"])
def version():
    """
    echo -e "DATE\tIP\tSVN\tSERIAL\tOP" > installs.log; chown scodoc  installs.log
    """
    remote_addr = request.environ.get("HTTP_X_REAL_IP", request.remote_addr)
    mode = request.args.get("mode", "?")
    sn = request.args.get("sn", "-1")  # serial number
    svn = request.args.get("svn", "")  # installed subversion (ScoDoc 7)
    release = request.args.get("release", "")  # ScoDoc 9 prod installs
    commit = request.args.get("commit", "")  # installed git commit (devs)
    if mode == "install" or not sn:
        serial = increment()
    else:
        serial = sn

    f = open(DIR + "installs.log", "a")
    f.write(
        "%s\t%s\t%s\t%s\t%s\n"
        % (
            time.strftime("%Y-%m-%d %H:%M:%S"),
            remote_addr,
            svn or commit or release or "-",
            serial,
            mode,
        )
    )
    f.close()
    return str(serial)