Michael Pilosov
6 months ago
commit
3c2d665bfd
14 changed files with 913 additions and 0 deletions
@ -0,0 +1,6 @@ |
|||||
|
out/ |
||||
|
*.png |
||||
|
*.gif |
||||
|
*.mp4 |
||||
|
__pycache__/ |
||||
|
*.db |
||||
@ -0,0 +1,51 @@ |
|||||
|
# Use Python 3.10.11 slim image as the base image |
||||
|
FROM python:3.10.11-slim |
||||
|
|
||||
|
# Set environment variables to avoid writing .pyc files and buffering stdout and stderr |
||||
|
ENV PYTHONDONTWRITEBYTECODE 1 |
||||
|
ENV PYTHONUNBUFFERED 1 |
||||
|
|
||||
|
# Create a new user 'user' with UID and GID of 1000 |
||||
|
RUN groupadd -g 1000 user && \ |
||||
|
useradd -m -s /bin/bash -u 1000 -g user user |
||||
|
|
||||
|
# Set environment variables for the user install |
||||
|
ENV PATH=/home/user/.local/bin:$PATH |
||||
|
|
||||
|
# Install system dependencies as root |
||||
|
RUN apt-get update && \ |
||||
|
apt-get install -y --no-install-recommends make ffmpeg dumb-init && \ |
||||
|
rm -rf /var/lib/apt/lists/* |
||||
|
|
||||
|
# Set the home directory |
||||
|
WORKDIR /home/user/ |
||||
|
RUN chown -R user:user /home/user |
||||
|
|
||||
|
# Switch to non-root user before copying files and installing Python packages |
||||
|
USER user |
||||
|
|
||||
|
# Copy the requirements file to /tmp and install Python dependencies with user flag |
||||
|
COPY --chown=user:user requirements.txt /tmp/requirements.txt |
||||
|
RUN python -m pip install --upgrade pip |
||||
|
RUN pip install --no-cache-dir --user -r /tmp/requirements.txt |
||||
|
|
||||
|
# APPLICATION SETUP |
||||
|
|
||||
|
# Copy the default profiles file and set the appropriate permissions |
||||
|
COPY --chown=user:user profiles.default.toml /home/user/.prefect/profiles.toml |
||||
|
|
||||
|
# Copy the application files |
||||
|
COPY --chown=user:user app ./app |
||||
|
COPY --chown=user:user noaa_animate.py . |
||||
|
COPY --chown=user:user start.sh . |
||||
|
COPY --chown=user:user init_db.py . |
||||
|
RUN chmod +x start.sh |
||||
|
RUN mkdir -p out |
||||
|
RUN python init_db.py /home/user/.prefect/prefect.db |
||||
|
|
||||
|
# Set the correct ownership (recursively) for /app |
||||
|
# Already owned by user due to --chown in COPY commands |
||||
|
|
||||
|
# Define the entrypoint and the commands to execute |
||||
|
ENTRYPOINT ["dumb-init", "--"] |
||||
|
CMD ["./start.sh"] |
||||
@ -0,0 +1,19 @@ |
|||||
|
# NOAA Animation Web App |
||||
|
 |
||||
|
|
||||
|
Animate images from a website full of urls by way of a proxy server and some Python functions wrapped with the `prefect` decorator. |
||||
|
Uses https://services.swpc.noaa.gov as an example. |
||||
|
|
||||
|
|
||||
|
## Instructions |
||||
|
If you have `docker` and `make` installed, just run: |
||||
|
|
||||
|
```bash |
||||
|
make |
||||
|
``` |
||||
|
|
||||
|
and visit `localhost:4200` to see Prefect, and `localhost:9021/iframe` to view the UI. |
||||
|
|
||||
|
An example Flow of about 100 images: |
||||
|
 |
||||
|
|
||||
@ -0,0 +1,150 @@ |
|||||
|
import logging |
||||
|
import os |
||||
|
import re |
||||
|
import time |
||||
|
from datetime import datetime |
||||
|
|
||||
|
import requests |
||||
|
from flask import Flask, Response, render_template, request, send_from_directory |
||||
|
from prefect.deployments import run_deployment |
||||
|
|
||||
|
PORT = 9021 |
||||
|
app = Flask(__name__) |
||||
|
|
||||
|
logging.basicConfig(level=logging.DEBUG) |
||||
|
|
||||
|
|
||||
|
def deploy_name(): |
||||
|
return datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S") + "Z" |
||||
|
|
||||
|
|
||||
|
def get_host(): |
||||
|
host = os.environ.get("LIGHTNING_CLOUDSPACE_HOST") |
||||
|
if host is None: |
||||
|
default_host = os.environ.get("HOST_NAME", "0.0.0.0") |
||||
|
return f"{default_host}:{PORT}" |
||||
|
else: |
||||
|
return f"{PORT}-{host}" |
||||
|
|
||||
|
|
||||
|
@app.route("/iframe") |
||||
|
@app.route("/iframe/") |
||||
|
@app.route("/iframe/<path:subpath>") |
||||
|
def home(subpath="images/animations/"): |
||||
|
host = get_host() |
||||
|
initial_url = f"http://{host}/{subpath}" |
||||
|
api_url = f"http://{host}/api" |
||||
|
return render_template( |
||||
|
"index.html", initial_url=initial_url, host=f"http://{host}", api_url=api_url |
||||
|
) |
||||
|
|
||||
|
|
||||
|
@app.route("/api", methods=["POST"]) |
||||
|
def handle_api(): |
||||
|
data = request.json # This assumes you're sending JSON data. |
||||
|
url = data.get("url") |
||||
|
if not url.endswith("/"): |
||||
|
url += "/" |
||||
|
|
||||
|
logging.debug(f"Received URL: {url}") |
||||
|
params = {"url": url, "limit": 24 * 60, "ext": None} |
||||
|
response = run_deployment( |
||||
|
name="create-animations/noaa-animate", |
||||
|
parameters=params, |
||||
|
flow_run_name=f"{deploy_name()}.webapp.{url}", |
||||
|
) |
||||
|
# response is a FlowRun - need to get what we want from it. |
||||
|
|
||||
|
# Process the data as needed. |
||||
|
return { |
||||
|
"status": "success", |
||||
|
"message": f"{url} processed successfully", |
||||
|
# "response": response, |
||||
|
}, 200 |
||||
|
|
||||
|
|
||||
|
@app.route("/videos/<path:filename>") |
||||
|
def custom_static(filename): |
||||
|
return send_from_directory("../out", filename) |
||||
|
|
||||
|
|
||||
|
@app.route("/", methods=["GET"]) |
||||
|
@app.route("/<path:url>", methods=["GET"]) |
||||
|
def proxy(url=""): |
||||
|
original_base_url = "https://services.swpc.noaa.gov" |
||||
|
host = get_host() |
||||
|
proxy_base_url = f"http://{host}/" |
||||
|
|
||||
|
target_url = f"{original_base_url}/{url}" |
||||
|
logging.debug(f"Fetching URL: {target_url}") |
||||
|
|
||||
|
try: |
||||
|
headers = { |
||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36" |
||||
|
} |
||||
|
response = requests.get(target_url, headers=headers, stream=True) |
||||
|
excluded_headers = [ |
||||
|
"content-encoding", |
||||
|
"content-length", |
||||
|
"transfer-encoding", |
||||
|
"connection", |
||||
|
] |
||||
|
headers = [ |
||||
|
(name, value) |
||||
|
for (name, value) in response.raw.headers.items() |
||||
|
if name.lower() not in excluded_headers |
||||
|
] |
||||
|
|
||||
|
if "text/html" in response.headers.get("Content-Type", ""): |
||||
|
content = response.content.decode("utf-8") |
||||
|
content = re.sub(r"'http://", "'https://", content) |
||||
|
content = re.sub( |
||||
|
r"https?://services.swpc.noaa.gov", proxy_base_url, content |
||||
|
) |
||||
|
|
||||
|
content = content.replace( |
||||
|
"</body>", |
||||
|
f""" |
||||
|
<script> |
||||
|
window.addEventListener('load', function() {{ |
||||
|
var hasSufficientImages = false; |
||||
|
var observer = new MutationObserver(function(mutations) {{ |
||||
|
mutations.forEach(function(mutation) {{ |
||||
|
if (mutation.type === 'childList') {{ |
||||
|
console.log('observer'); |
||||
|
checkImages(); |
||||
|
}} |
||||
|
}}); |
||||
|
}}); |
||||
|
observer.observe(document.body, {{ childList: true, subtree: true }}); |
||||
|
|
||||
|
function checkImages() {{ |
||||
|
var links = document.querySelectorAll('a'); |
||||
|
var imageLinkRegex = /\.(jpg|jpeg|png)(\\?.*)?(#.*)?$/i; |
||||
|
var numImages = Array.from(links).filter(link => imageLinkRegex.test(link.href)).length; |
||||
|
console.log('Number of eligible links:', numImages); |
||||
|
hasSufficientImages = numImages >= 60; |
||||
|
window.parent.postMessage({{ type: 'urlUpdate', url: '{original_base_url}/{url}', eligible: hasSufficientImages }}, '*'); |
||||
|
}} |
||||
|
|
||||
|
// Run initial check in case content is already loaded |
||||
|
console.log('initial'); |
||||
|
checkImages(); |
||||
|
}}); |
||||
|
</script> |
||||
|
</body>""", |
||||
|
) |
||||
|
content = content.encode("utf-8") |
||||
|
return Response(content, status=response.status_code, headers=headers) |
||||
|
else: |
||||
|
return Response( |
||||
|
response.content, status=response.status_code, headers=headers |
||||
|
) |
||||
|
|
||||
|
except Exception as e: |
||||
|
logging.error(f"Error fetching URL: {e}") |
||||
|
return Response(f"Error fetching URL: {e}", status=500) |
||||
|
|
||||
|
|
||||
|
if __name__ == "__main__": |
||||
|
app.run(host="0.0.0.0", port=9021, debug=True) |
||||
@ -0,0 +1,5 @@ |
|||||
|
start: |
||||
|
gunicorn --worker-class gevent --bind 0.0.0.0:9021 app:app |
||||
|
|
||||
|
dev: |
||||
|
python app.py |
||||
@ -0,0 +1,271 @@ |
|||||
|
<!DOCTYPE html> |
||||
|
<html lang="en"> |
||||
|
|
||||
|
<head> |
||||
|
<meta charset="UTF-8"> |
||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0"> |
||||
|
<title>Animator</title> |
||||
|
<style> |
||||
|
body { |
||||
|
font-family: Arial, sans-serif; |
||||
|
display: flex; |
||||
|
flex-direction: column; |
||||
|
align-items: center; |
||||
|
margin: 0; |
||||
|
padding: 20px; |
||||
|
} |
||||
|
|
||||
|
iframe { |
||||
|
width: 99%; |
||||
|
/* Full width to use available space */ |
||||
|
height: 500px; |
||||
|
border: 1px solid #ccc; |
||||
|
margin-bottom: 20px; |
||||
|
} |
||||
|
|
||||
|
#current-url { |
||||
|
font-size: 16px; |
||||
|
color: #333; |
||||
|
width: 100%; |
||||
|
/* Makes sure it takes the full container width */ |
||||
|
word-wrap: break-word; |
||||
|
/* Ensures the text wraps in the div */ |
||||
|
margin-bottom: 10px; |
||||
|
/* Space before the navigation buttons */ |
||||
|
} |
||||
|
|
||||
|
.button-group { |
||||
|
display: flex; |
||||
|
/* Aligns buttons in a row */ |
||||
|
justify-content: center; |
||||
|
/* Centers buttons horizontally */ |
||||
|
gap: 10px; |
||||
|
/* Adds space between buttons */ |
||||
|
} |
||||
|
|
||||
|
button { |
||||
|
padding: 5px 10px; |
||||
|
margin: 10px; |
||||
|
border-radius: 5px; |
||||
|
/* Rounded corners */ |
||||
|
border: 1px solid #ccc; |
||||
|
/* Grey border */ |
||||
|
display: inline-block; |
||||
|
/* Ensures buttons are inline and can control additional layout properties */ |
||||
|
font-size: 16px; |
||||
|
} |
||||
|
|
||||
|
#backButton, |
||||
|
#forwardButton { |
||||
|
background-color: #f0f0f0; |
||||
|
/* Light grey background */ |
||||
|
color: #333; |
||||
|
/* Dark text */ |
||||
|
padding: 0px 10px; |
||||
|
/* Reduced vertical padding for narrow height */ |
||||
|
cursor: pointer; |
||||
|
/* Cursor indicates button */ |
||||
|
height: 24px; |
||||
|
/* Fixed height for a narrower button */ |
||||
|
line-height: 16px; |
||||
|
/* Adjust line height to vertically center the text */ |
||||
|
margin: 2px; |
||||
|
/* Small margin to separate buttons slightly */ |
||||
|
} |
||||
|
|
||||
|
#backButton:hover, |
||||
|
#forwardButton:hover { |
||||
|
background-color: #e8e8e8; |
||||
|
/* Slightly darker background on hover */ |
||||
|
} |
||||
|
|
||||
|
video { |
||||
|
display: none; |
||||
|
/* Initially hide the video player */ |
||||
|
width: 99%; |
||||
|
/* Adjust based on your layout needs, or use max-width for responsiveness */ |
||||
|
height: auto; |
||||
|
/* Maintain aspect ratio */ |
||||
|
margin-top: 20px; |
||||
|
/* Ensure it's centered properly */ |
||||
|
max-width: 640px; |
||||
|
/* Max width of the video */ |
||||
|
border: 1px solid #ccc; |
||||
|
/* Optional, adds a border for better visibility */ |
||||
|
} |
||||
|
</style> |
||||
|
</head> |
||||
|
|
||||
|
<body> |
||||
|
<h1>Animate a Folder of Images</h1> |
||||
|
<p>Navigate to a folder of 60+ images.</p> |
||||
|
<iframe id="iframe" src="{{ initial_url }}"></iframe> |
||||
|
<div class="button-group"> <!-- Button group for inline display --> |
||||
|
<button id="backButton" onclick="goBack()">←</button> |
||||
|
<button id="forwardButton" onclick="goForward()">→</button> |
||||
|
</div> |
||||
|
<button id="submit-button" onclick="submitUrl()" style="display:none;">Create Latest Movie</button> |
||||
|
<div id="loading-spinner" style="display: none;"> |
||||
|
<div class="spinner"></div> |
||||
|
</div> |
||||
|
|
||||
|
<style> |
||||
|
.spinner { |
||||
|
border: 8px solid #f3f3f3; |
||||
|
/* Light grey */ |
||||
|
border-top: 8px solid #3498db; |
||||
|
/* Blue */ |
||||
|
border-radius: 50%; |
||||
|
width: 50px; |
||||
|
height: 50px; |
||||
|
animation: spin 45s linear infinite; |
||||
|
} |
||||
|
|
||||
|
@keyframes spin { |
||||
|
0% { |
||||
|
transform: rotate(0deg); |
||||
|
} |
||||
|
|
||||
|
100% { |
||||
|
transform: rotate(360deg); |
||||
|
} |
||||
|
} |
||||
|
</style> |
||||
|
|
||||
|
<video id="video-player" controls loop style="display: none;"> |
||||
|
<source id="video-source" type="video/mp4"> |
||||
|
Your browser does not support the video tag. |
||||
|
</video> |
||||
|
<script> |
||||
|
function goBack() { |
||||
|
document.getElementById('iframe').contentWindow.history.back(); |
||||
|
} |
||||
|
|
||||
|
function goForward() { |
||||
|
document.getElementById('iframe').contentWindow.history.forward(); |
||||
|
} |
||||
|
|
||||
|
// function updateUrl(url) { |
||||
|
// document.getElementById('url').textContent = url; |
||||
|
// } |
||||
|
|
||||
|
function updateUrl(url) { |
||||
|
document.getElementById('url').textContent = url; |
||||
|
} |
||||
|
|
||||
|
function handleVideoSuccess() { |
||||
|
console.log("Video loaded successfully."); |
||||
|
document.getElementById('video-player').style.display = 'block'; // Show the video player only if the video loads successfully |
||||
|
} |
||||
|
|
||||
|
function handleVideoError() { |
||||
|
console.log("Unable to load video."); |
||||
|
document.getElementById('video-player').style.display = 'none'; // Hide the video player |
||||
|
document.getElementById('submit-button').textContent = 'Generate Movie'; |
||||
|
} |
||||
|
|
||||
|
function updateVideo(url) { |
||||
|
// Convert the full URL to a format suitable for your video path |
||||
|
let formattedPath = url.replace(/https?:\/\//, '') // Remove the protocol part |
||||
|
.replace(/\./g, '_') // Replace dots with underscores |
||||
|
.replace(/\//g, '-'); // Replace slashes with hyphens |
||||
|
// Check if the formattedPath ends with a slash, if not append '-' |
||||
|
if (!formattedPath.endsWith('-')) { |
||||
|
formattedPath += '-'; |
||||
|
} |
||||
|
|
||||
|
// Append '.mp4' to the formatted path |
||||
|
let videoPath = `${formattedPath}latest.mp4`; |
||||
|
let videoPlayer = document.getElementById('video-player'); |
||||
|
let videoSource = document.getElementById('video-source'); |
||||
|
|
||||
|
videoPlayer.muted = true; |
||||
|
// Set up event listeners before setting the source |
||||
|
videoSource.onerror = handleVideoError; |
||||
|
// videoSource.onloadedmetadata = handleVideoSuccess; |
||||
|
|
||||
|
console.log("Fetched latest") |
||||
|
videoSource.src = `/videos/${videoPath}`; |
||||
|
videoPlayer.load(); |
||||
|
// videoPlayer.style.display = 'block'; |
||||
|
videoPlayer.play().then(() => { |
||||
|
// The video is playing, show the player |
||||
|
console.log("Video loaded and playing."); |
||||
|
videoPlayer.style.display = 'block'; |
||||
|
}).catch(error => { |
||||
|
// Error playing the video |
||||
|
console.log("Failed to play video: ", error); |
||||
|
videoPlayer.style.display = 'none'; |
||||
|
}); |
||||
|
document.getElementById('submit-button').textContent = 'Generate Latest Movie'; |
||||
|
} |
||||
|
|
||||
|
window.addEventListener('message', function (event) { |
||||
|
if (event.origin === '{{ host }}') { |
||||
|
var data = event.data; |
||||
|
if (data && data.type === 'urlUpdate') { |
||||
|
const submitButton = document.getElementById('submit-button'); |
||||
|
const videoPlayer = document.getElementById('video-player'); |
||||
|
updateUrl(data.url); |
||||
|
if (data.eligible) { |
||||
|
submitButton.style.display = 'block'; // Show the button |
||||
|
updateVideo(data.url); |
||||
|
} else { |
||||
|
submitButton.style.display = 'none'; // Hide the button |
||||
|
videoPlayer.style.display = 'none'; // Hide the video |
||||
|
} |
||||
|
const newSubpath = new URL(data.url).pathname; // Extract the path from the URL |
||||
|
// Update the browser's URL to reflect the iframe's navigation |
||||
|
const newPath = `/iframe${newSubpath}`; // Construct the new path |
||||
|
document.getElementById('share-button').setAttribute('data-url', window.location.origin + newPath); |
||||
|
// history.pushState({ path: newPath }, '', newPath); |
||||
|
} |
||||
|
} |
||||
|
}); |
||||
|
|
||||
|
function submitUrl() { |
||||
|
const url = document.getElementById('url').textContent; |
||||
|
const payload = { url: url }; |
||||
|
document.getElementById('loading-spinner').style.display = 'block'; // Show the loading spinner |
||||
|
document.getElementById('submit-button').style.display = 'none'; // Hide the button |
||||
|
console.log("Requesting new video.") |
||||
|
fetch('{{ api_url }}', { |
||||
|
method: 'POST', |
||||
|
headers: { |
||||
|
'Content-Type': 'application/json' |
||||
|
}, |
||||
|
body: JSON.stringify(payload) |
||||
|
}).then(response => response.json()) |
||||
|
.then(data => { |
||||
|
console.log(data); |
||||
|
// Hide the loading spinner |
||||
|
document.getElementById('loading-spinner').style.display = 'none'; |
||||
|
document.getElementById('submit-button').style.display = 'block'; |
||||
|
updateUrl(url); |
||||
|
// Re-attempt to load the video |
||||
|
updateVideo(url); |
||||
|
}) |
||||
|
.catch(error => { |
||||
|
console.error('Error:', error); |
||||
|
// Hide the loading spinner |
||||
|
document.getElementById('loading-spinner').style.display = 'none'; |
||||
|
document.getElementById('submit-button').style.display = 'block'; |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
function copyUrlToClipboard() { |
||||
|
const url = document.getElementById('share-button').getAttribute('data-url'); |
||||
|
navigator.clipboard.writeText(url).then(() => { |
||||
|
alert('URL copied to clipboard!'); |
||||
|
}).catch(err => { |
||||
|
console.error('Failed to copy: ', err); |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
</script> |
||||
|
<button id="share-button" onclick="copyUrlToClipboard()">Share Link</button> |
||||
|
|
||||
|
<div align="center" id="current-url">Source: <span id="url">Loading...</span></div> |
||||
|
</body> |
||||
|
|
||||
|
</html> |
||||
@ -0,0 +1,97 @@ |
|||||
|
import argparse |
||||
|
import logging |
||||
|
import os |
||||
|
import sqlite3 |
||||
|
|
||||
|
# Setup basic configuration for logging |
||||
|
logging.basicConfig( |
||||
|
level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" |
||||
|
) |
||||
|
|
||||
|
|
||||
|
def initialize_db(db_path): |
||||
|
# Check if the database file already exists |
||||
|
db_exists = os.path.exists(db_path) |
||||
|
if db_exists: |
||||
|
logging.info(f"{db_path} exists") # Log a message if the database exists |
||||
|
return |
||||
|
try: |
||||
|
with sqlite3.connect( |
||||
|
db_path |
||||
|
) as conn: # Using 'with' to ensure that the connection is closed automatically |
||||
|
configure_database(conn) |
||||
|
except sqlite3.Error as e: |
||||
|
logging.error(f"Database error: {e}") # Log any SQLite errors that occur |
||||
|
except Exception as e: |
||||
|
logging.error( |
||||
|
f"Exception in initialize_db: {e}" |
||||
|
) # Log any other exceptions that occur |
||||
|
|
||||
|
|
||||
|
def configure_database(conn): |
||||
|
cursor = conn.cursor() |
||||
|
# Setting the journal mode to WAL for better concurrency |
||||
|
cursor.execute("PRAGMA journal_mode = WAL;") |
||||
|
# Setting synchronous to NORMAL for a balance between speed and reliability |
||||
|
cursor.execute("PRAGMA synchronous = NORMAL;") |
||||
|
# Setting a busy timeout to prevent immediate failures when the database is locked |
||||
|
cursor.execute("PRAGMA busy_timeout = 5000;") |
||||
|
# Increasing the cache size to reduce the number of disk I/O operations |
||||
|
cursor.execute("PRAGMA cache_size = -32000;") |
||||
|
# Enabling memory-mapped I/O for potentially faster file operations |
||||
|
cursor.execute("PRAGMA mmap_size = 536870912;") |
||||
|
# Setting locking mode to EXCLUSIVE can enhance performance for single-user scenarios |
||||
|
cursor.execute("PRAGMA locking_mode = EXCLUSIVE;") |
||||
|
# Ensuring foreign key constraints are enforced for data integrity |
||||
|
cursor.execute("PRAGMA foreign_keys = ON;") |
||||
|
conn.commit() # Commit all PRAGMA configurations |
||||
|
|
||||
|
logging.info("Set up database with multi-user optimizations.") |
||||
|
|
||||
|
|
||||
|
def batch_transact(db_path, operations): |
||||
|
try: |
||||
|
with sqlite3.connect( |
||||
|
db_path |
||||
|
) as conn: # Ensure that the connection is handled properly |
||||
|
cursor = conn.cursor() |
||||
|
cursor.execute( |
||||
|
"BEGIN TRANSACTION;" |
||||
|
) # Start a transaction for batch operations |
||||
|
for operation in operations: |
||||
|
cursor.execute( |
||||
|
operation |
||||
|
) # Execute each SQL operation provided in the operations list |
||||
|
cursor.execute("COMMIT;") # Commit all operations at once |
||||
|
except sqlite3.Error as e: |
||||
|
logging.error(f"Database error during batch transaction: {e}") |
||||
|
except Exception as e: |
||||
|
logging.error(f"Exception in batch_transact: {e}") |
||||
|
|
||||
|
|
||||
|
def maintenance(db_path): |
||||
|
try: |
||||
|
with sqlite3.connect(db_path) as conn: |
||||
|
cursor = conn.cursor() |
||||
|
cursor.execute( |
||||
|
"PRAGMA optimize;" |
||||
|
) # Optimize the database to maintain performance |
||||
|
cursor.execute("VACUUM;") # Reclaim space and defragment the database file |
||||
|
except sqlite3.Error as e: |
||||
|
logging.error(f"Database error during maintenance: {e}") |
||||
|
except Exception as e: |
||||
|
logging.error(f"Exception in maintenance: {e}") |
||||
|
|
||||
|
|
||||
|
def parse_args(): |
||||
|
parser = argparse.ArgumentParser( |
||||
|
description="Initialize and manage an SQLite database." |
||||
|
) |
||||
|
parser.add_argument("db_path", type=str, help="Path to the SQLite database file.") |
||||
|
args = parser.parse_args() |
||||
|
return args |
||||
|
|
||||
|
|
||||
|
if __name__ == "__main__": |
||||
|
args = parse_args() # Parse the command-line arguments for the database path |
||||
|
initialize_db(args.db_path) # Use the parsed path to initialize the database |
||||
@ -0,0 +1,10 @@ |
|||||
|
run: build |
||||
|
docker run --rm -ti --name noaa -e HOST_NAME=localhost -p 9021:9021 -p 4200:4200 noaa |
||||
|
|
||||
|
build: |
||||
|
docker build -t noaa . |
||||
|
|
||||
|
lint: |
||||
|
isort --profile=black . |
||||
|
black . |
||||
|
|
||||
@ -0,0 +1,266 @@ |
|||||
|
import os |
||||
|
import re |
||||
|
from datetime import datetime, timedelta |
||||
|
from io import BytesIO |
||||
|
from typing import Dict, Iterator, List |
||||
|
|
||||
|
import httpx |
||||
|
# import imageio |
||||
|
import numpy as np |
||||
|
from moviepy.editor import ImageSequenceClip |
||||
|
from PIL import Image |
||||
|
from prefect import flow, task |
||||
|
from prefect.task_runners import ConcurrentTaskRunner |
||||
|
from prefect.tasks import task_input_hash |
||||
|
|
||||
|
BASE_URL = "https://services.swpc.noaa.gov/images/animations/geospace/" |
||||
|
|
||||
|
|
||||
|
@task( |
||||
|
retries=3, |
||||
|
retry_delay_seconds=5, |
||||
|
cache_key_fn=task_input_hash, |
||||
|
cache_expiration=timedelta(minutes=2), |
||||
|
log_prints=True, |
||||
|
) |
||||
|
def get_file_links(url: str, ext: str | None = None) -> Iterator[str]: |
||||
|
response = httpx.get(url) |
||||
|
response.raise_for_status() |
||||
|
webpage_content = response.text |
||||
|
if ext is None: |
||||
|
print("Extension not supplied. Inferring (less efficient) png/jpg/jpeg") |
||||
|
exts = ["png", "jpg", "jpeg"] |
||||
|
else: |
||||
|
exts = [ext.lower()] |
||||
|
lines = webpage_content.split("\n") |
||||
|
for line in lines: |
||||
|
for ext in exts: |
||||
|
if ext in line: # need to parse the href link |
||||
|
start_pos = line.find('href="') + len('href="') |
||||
|
end_pos = line.find('"', start_pos) |
||||
|
href = line[start_pos:end_pos] |
||||
|
if href.endswith(f"latest.{ext}"): |
||||
|
print("Skipping latest") |
||||
|
continue |
||||
|
if href.endswith(ext): |
||||
|
if not href.startswith("http"): |
||||
|
href = url + href |
||||
|
yield href |
||||
|
break # Exit the inner loop to avoid duplicate yields for multiple exts |
||||
|
|
||||
|
|
||||
|
def url_tail_hash(context, parameters): |
||||
|
# return a constant |
||||
|
return parameters["url"].split("/")[-1] |
||||
|
|
||||
|
|
||||
|
def out_path_hash(context, parameters): |
||||
|
return parameters["output_path"] + f"_L{len(parameters['images'])}" |
||||
|
|
||||
|
|
||||
|
@task( |
||||
|
retries=5, |
||||
|
retry_delay_seconds=1, |
||||
|
cache_key_fn=task_input_hash, |
||||
|
cache_expiration=timedelta(minutes=5), |
||||
|
result_storage_key="{parameters[url]}", |
||||
|
) |
||||
|
def get_content(url: str, params: Dict[str, any] | None = None): |
||||
|
response = httpx.get(f"https://{url}", params=params) |
||||
|
try: |
||||
|
response.raise_for_status() |
||||
|
return response.content |
||||
|
except httpx.HTTPStatusError: |
||||
|
return None |
||||
|
|
||||
|
|
||||
|
def preview_urls(urls): |
||||
|
print("URLS (head):") |
||||
|
print(urls[:5]) |
||||
|
print("URLS (tail):") |
||||
|
print(urls[-5:]) |
||||
|
|
||||
|
|
||||
|
@task( |
||||
|
cache_key_fn=task_input_hash, |
||||
|
cache_expiration=timedelta(hours=1), |
||||
|
) |
||||
|
def get_images(urls: List[str] | List[str], limit: int = 0): |
||||
|
if limit > 0: |
||||
|
print(f"Limiting to {limit} urls") |
||||
|
urls = urls[-limit:] |
||||
|
|
||||
|
urls = [url.replace("https://", "").replace("http://", "") for url in urls] |
||||
|
preview_urls(urls) |
||||
|
|
||||
|
futures = get_content.map(urls) |
||||
|
images = [ |
||||
|
(urls[i], f.result()) for i, f in enumerate(futures) if f.result() is not None |
||||
|
] |
||||
|
return images |
||||
|
|
||||
|
|
||||
|
def extract_timestamp_from_url(url: str) -> str: |
||||
|
# Assuming the timestamp format is in the format shown in the screenshot |
||||
|
match = re.search(r"\d{8}_\d{6}", url) |
||||
|
return match.group(0) if match else "" |
||||
|
|
||||
|
|
||||
|
# @task( |
||||
|
# cache_key_fn=out_path_hash, |
||||
|
# cache_expiration=timedelta(minutes=3), |
||||
|
# result_storage_key="{parameters[output_path]}", |
||||
|
# ) |
||||
|
# def create_animation( |
||||
|
# images: List[bytes], output_path: str, duration: float = 0.5 |
||||
|
# ) -> None: |
||||
|
# if not images: |
||||
|
# raise ValueError("No images!") |
||||
|
# pil_images = [Image.open(BytesIO(img_data)).convert("RGB") for img_data in images] |
||||
|
# imageio.mimsave(output_path, pil_images, duration=duration) |
||||
|
# return output_path |
||||
|
|
||||
|
|
||||
|
def make_even_dimensions(image): |
||||
|
width, height = image.size |
||||
|
if width % 2 == 1: |
||||
|
width -= 1 |
||||
|
if height % 2 == 1: |
||||
|
height -= 1 |
||||
|
return image.resize((width, height), Image.ANTIALIAS) |
||||
|
|
||||
|
|
||||
|
def crop_to_even(image): |
||||
|
width, height = image.size |
||||
|
# Adjust width and height to be even |
||||
|
if width % 2 == 1: |
||||
|
width -= 1 |
||||
|
if height % 2 == 1: |
||||
|
height -= 1 |
||||
|
return image.crop((0, 0, width, height)) |
||||
|
|
||||
|
|
||||
|
@task( |
||||
|
cache_key_fn=out_path_hash, |
||||
|
cache_expiration=timedelta(hours=4), |
||||
|
result_storage_key="{parameters[output_path]}", |
||||
|
) |
||||
|
def create_mp4_animation(images: List[bytes], output_path: str, fps: int = 24) -> None: |
||||
|
# Convert bytes to PIL images and then to numpy arrays |
||||
|
frames = [ |
||||
|
np.array(crop_to_even(Image.open(BytesIO(img_data)).convert("RGB"))) |
||||
|
for img_data in images |
||||
|
] |
||||
|
|
||||
|
# Create a video clip from the image sequence |
||||
|
clip = ImageSequenceClip(frames, fps=fps) |
||||
|
|
||||
|
# Write the video clip to a file |
||||
|
clip.write_videofile( |
||||
|
output_path, |
||||
|
codec="libx264", |
||||
|
ffmpeg_params=["-pix_fmt", "yuv420p"], |
||||
|
preset="medium", |
||||
|
bitrate="800k", |
||||
|
) |
||||
|
|
||||
|
return output_path |
||||
|
|
||||
|
|
||||
|
def format_output_name(url: str, latest: bool = False): |
||||
|
if latest: |
||||
|
now = "latest" |
||||
|
else: |
||||
|
now = datetime.now().strftime("%Y%m%d-%H:%M:%S") |
||||
|
return ( |
||||
|
url.replace("https://", "") |
||||
|
.replace("http://", "") |
||||
|
.replace("/", "-") |
||||
|
.replace(".", "_") |
||||
|
+ now |
||||
|
) |
||||
|
|
||||
|
|
||||
|
@task( |
||||
|
name="animate", |
||||
|
retries=0, |
||||
|
retry_delay_seconds=1, |
||||
|
log_prints=True, |
||||
|
cache_key_fn=task_input_hash, |
||||
|
cache_expiration=timedelta(minutes=3), |
||||
|
) |
||||
|
def animate( |
||||
|
url: str = "https://services.swpc.noaa.gov/images/animations/geospace/density/", |
||||
|
ext: str = "png", |
||||
|
latest: bool = True, |
||||
|
limit: int = 0, |
||||
|
): |
||||
|
urls = get_file_links(url, ext) |
||||
|
if len(urls) == 0: |
||||
|
raise ValueError("No urls scraped") |
||||
|
images = get_images(list(sorted(urls)), limit=limit) |
||||
|
if len(images) == 0: |
||||
|
raise ValueError("No images retrieved.") |
||||
|
print(f"Retrieved {len(images)} images.") |
||||
|
sorted_images = sorted(images, key=lambda x: extract_timestamp_from_url(x[0])) |
||||
|
print("Head:") |
||||
|
print([u for u, i in sorted_images[:5]]) |
||||
|
frames = [s[1] for s in sorted_images] |
||||
|
# create_animation(frames, "out.gif", duration=5) |
||||
|
out_name = format_output_name(url, latest=latest) |
||||
|
create_mp4_animation(frames, f"out/{out_name}.mp4") |
||||
|
|
||||
|
|
||||
|
def deploy_name(): |
||||
|
return datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S") + "Z" |
||||
|
|
||||
|
|
||||
|
@flow( |
||||
|
name="create-animations", |
||||
|
retries=0, |
||||
|
retry_delay_seconds=1, |
||||
|
log_prints=True, |
||||
|
task_runner=ConcurrentTaskRunner(), |
||||
|
flow_run_name=None, |
||||
|
timeout_seconds=90, |
||||
|
) |
||||
|
def create_animations( |
||||
|
url: str | List[str] = BASE_URL + "velocity/", |
||||
|
ext: str | None = None, |
||||
|
latest: bool = False, |
||||
|
limit: int = 0, |
||||
|
): |
||||
|
if isinstance(url, str): |
||||
|
url = [url] |
||||
|
|
||||
|
futures = animate.map(url, ext, latest, limit) |
||||
|
return futures |
||||
|
|
||||
|
|
||||
|
if __name__ == "__main__": |
||||
|
# make_animation.from_source( |
||||
|
# source=TEST_REPO, |
||||
|
# entrypoint="noaa_animate.py:make_animation", |
||||
|
# ).deploy( |
||||
|
# name="noaa-animate", work_pool_name="process" |
||||
|
# ) |
||||
|
|
||||
|
from prefect.client.schemas.schedules import CronSchedule |
||||
|
|
||||
|
sched = CronSchedule(cron="*/15 * * * *", timezone="America/Denver") |
||||
|
|
||||
|
links = [ |
||||
|
BASE_URL + "density/", |
||||
|
BASE_URL + "velocity/", |
||||
|
BASE_URL + "pressure/", |
||||
|
] |
||||
|
sched_params = { |
||||
|
"latest": True, |
||||
|
"url": links, |
||||
|
"ext": "png", |
||||
|
"limit": 0, |
||||
|
} |
||||
|
create_animations.serve( |
||||
|
"noaa-animate", limit=8, schedule=None, parameters=sched_params |
||||
|
) |
||||
|
# make_animation(url) |
||||
|
After Width: | Height: | Size: 448 KiB |
|
After Width: | Height: | Size: 408 KiB |
@ -0,0 +1,14 @@ |
|||||
|
active = "default" |
||||
|
PREFECT_API_URL = "http://0.0.0.0:4200/api" |
||||
|
|
||||
|
[profiles.default] |
||||
|
PREFECT_TASK_SCHEDULING_MAX_SCHEDULED_QUEUE_SIZE = 4 |
||||
|
|
||||
|
PREFECT_API_SERVICES_SCHEDULER_DEPLOYMENT_BATCH_SIZE = 100 |
||||
|
PREFECT_API_SERVICES_SCHEDULER_ENABLED = true |
||||
|
PREFECT_API_SERVICES_SCHEDULER_INSERT_BATCH_SIZE = 500 |
||||
|
PREFECT_API_SERVICES_SCHEDULER_LOOP_SECONDS = 60 |
||||
|
PREFECT_API_SERVICES_SCHEDULER_MIN_RUNS = 3 |
||||
|
PREFECT_API_SERVICES_SCHEDULER_MAX_RUNS = 100 |
||||
|
PREFECT_API_SERVICES_SCHEDULER_MIN_SCHEDULED_TIME = '0:30:00' |
||||
|
PREFECT_API_SERVICES_SCHEDULER_MAX_SCHEDULED_TIME = '0 days, 8:00:00' |
||||
@ -0,0 +1,9 @@ |
|||||
|
prefect==2.17.1 |
||||
|
Flask==3.0.3 |
||||
|
gunicorn==22.0.0 |
||||
|
gevent==24.2.1 |
||||
|
moviepy==1.0.3 |
||||
|
pillow==10.3.0 |
||||
|
requests==2.32.3 |
||||
|
httpx==0.27.0 |
||||
|
# imageio==2.34.1 |
||||
@ -0,0 +1,15 @@ |
|||||
|
#!/bin/bash |
||||
|
# Start the web app |
||||
|
cd app && make & |
||||
|
|
||||
|
# Start Prefect in the background |
||||
|
prefect server start --host 0.0.0.0 & |
||||
|
|
||||
|
sleep 10 |
||||
|
|
||||
|
# Start the deployment |
||||
|
python noaa_animate.py & |
||||
|
|
||||
|
# Wait for all background jobs to finish |
||||
|
wait |
||||
|
|
||||
Loading…
Reference in new issue