From 3c2d665bfd4ceef056c183e0ad13f3899bb77a3f Mon Sep 17 00:00:00 2001 From: Michael Pilosov
Date: Fri, 31 May 2024 11:25:40 -0600 Subject: [PATCH] new repo --- .gitignore | 6 + Dockerfile | 51 ++++++++ README.md | 19 +++ app/app.py | 150 ++++++++++++++++++++++ app/makefile | 5 + app/templates/index.html | 271 +++++++++++++++++++++++++++++++++++++++ init_db.py | 97 ++++++++++++++ makefile | 10 ++ noaa_animate.py | 266 ++++++++++++++++++++++++++++++++++++++ prefect.jpg | Bin 0 -> 458684 bytes preview.jpg | Bin 0 -> 417876 bytes profiles.default.toml | 14 ++ requirements.txt | 9 ++ start.sh | 15 +++ 14 files changed, 913 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 app/app.py create mode 100644 app/makefile create mode 100644 app/templates/index.html create mode 100644 init_db.py create mode 100644 makefile create mode 100644 noaa_animate.py create mode 100644 prefect.jpg create mode 100644 preview.jpg create mode 100644 profiles.default.toml create mode 100644 requirements.txt create mode 100755 start.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..424d620 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +out/ +*.png +*.gif +*.mp4 +__pycache__/ +*.db diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f984865 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,51 @@ +# Use Python 3.10.11 slim image as the base image +FROM python:3.10.11-slim + +# Set environment variables to avoid writing .pyc files and buffering stdout and stderr +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +# Create a new user 'user' with UID and GID of 1000 +RUN groupadd -g 1000 user && \ + useradd -m -s /bin/bash -u 1000 -g user user + +# Set environment variables for the user install +ENV PATH=/home/user/.local/bin:$PATH + +# Install system dependencies as root +RUN apt-get update && \ + apt-get install -y --no-install-recommends make ffmpeg dumb-init && \ + rm -rf /var/lib/apt/lists/* + +# Set the home directory +WORKDIR /home/user/ +RUN chown -R user:user /home/user + +# Switch to non-root user before copying files and installing Python packages +USER user + +# Copy the requirements file to /tmp and install Python dependencies with user flag +COPY --chown=user:user requirements.txt /tmp/requirements.txt +RUN python -m pip install --upgrade pip +RUN pip install --no-cache-dir --user -r /tmp/requirements.txt + +# APPLICATION SETUP + +# Copy the default profiles file and set the appropriate permissions +COPY --chown=user:user profiles.default.toml /home/user/.prefect/profiles.toml + +# Copy the application files +COPY --chown=user:user app ./app +COPY --chown=user:user noaa_animate.py . +COPY --chown=user:user start.sh . +COPY --chown=user:user init_db.py . +RUN chmod +x start.sh +RUN mkdir -p out +RUN python init_db.py /home/user/.prefect/prefect.db + +# Set the correct ownership (recursively) for /app +# Already owned by user due to --chown in COPY commands + +# Define the entrypoint and the commands to execute +ENTRYPOINT ["dumb-init", "--"] +CMD ["./start.sh"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..17d748d --- /dev/null +++ b/README.md @@ -0,0 +1,19 @@ +# NOAA Animation Web App + + +Animate images from a website full of urls by way of a proxy server and some Python functions wrapped with the `prefect` decorator. +Uses https://services.swpc.noaa.gov as an example. + + +## Instructions +If you have `docker` and `make` installed, just run: + +```bash +make +``` + +and visit `localhost:4200` to see Prefect, and `localhost:9021/iframe` to view the UI. + +An example Flow of about 100 images: + + diff --git a/app/app.py b/app/app.py new file mode 100644 index 0000000..1fc7303 --- /dev/null +++ b/app/app.py @@ -0,0 +1,150 @@ +import logging +import os +import re +import time +from datetime import datetime + +import requests +from flask import Flask, Response, render_template, request, send_from_directory +from prefect.deployments import run_deployment + +PORT = 9021 +app = Flask(__name__) + +logging.basicConfig(level=logging.DEBUG) + + +def deploy_name(): + return datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S") + "Z" + + +def get_host(): + host = os.environ.get("LIGHTNING_CLOUDSPACE_HOST") + if host is None: + default_host = os.environ.get("HOST_NAME", "0.0.0.0") + return f"{default_host}:{PORT}" + else: + return f"{PORT}-{host}" + + +@app.route("/iframe") +@app.route("/iframe/") +@app.route("/iframe/
") +def home(subpath="images/animations/"): + host = get_host() + initial_url = f"http://{host}/{subpath}" + api_url = f"http://{host}/api" + return render_template( + "index.html", initial_url=initial_url, host=f"http://{host}", api_url=api_url + ) + + +@app.route("/api", methods=["POST"]) +def handle_api(): + data = request.json # This assumes you're sending JSON data. + url = data.get("url") + if not url.endswith("/"): + url += "/" + + logging.debug(f"Received URL: {url}") + params = {"url": url, "limit": 24 * 60, "ext": None} + response = run_deployment( + name="create-animations/noaa-animate", + parameters=params, + flow_run_name=f"{deploy_name()}.webapp.{url}", + ) + # response is a FlowRun - need to get what we want from it. + + # Process the data as needed. + return { + "status": "success", + "message": f"{url} processed successfully", + # "response": response, + }, 200 + + +@app.route("/videos/
") +def custom_static(filename): + return send_from_directory("../out", filename) + + +@app.route("/", methods=["GET"]) +@app.route("/
", methods=["GET"]) +def proxy(url=""): + original_base_url = "https://services.swpc.noaa.gov" + host = get_host() + proxy_base_url = f"http://{host}/" + + target_url = f"{original_base_url}/{url}" + logging.debug(f"Fetching URL: {target_url}") + + try: + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36" + } + response = requests.get(target_url, headers=headers, stream=True) + excluded_headers = [ + "content-encoding", + "content-length", + "transfer-encoding", + "connection", + ] + headers = [ + (name, value) + for (name, value) in response.raw.headers.items() + if name.lower() not in excluded_headers + ] + + if "text/html" in response.headers.get("Content-Type", ""): + content = response.content.decode("utf-8") + content = re.sub(r"'http://", "'https://", content) + content = re.sub( + r"https?://services.swpc.noaa.gov", proxy_base_url, content + ) + + content = content.replace( + "