airano-ir
f203ca88de
Three-month batch sync from internal repo (~80 commits) covering Tracks F.5a, F.7e, F.8, F.17, F.18, F.X. WordPress media pipeline - Pillow-based optimization, AI image generation (OpenAI / Stability / Replicate / Google Nano Banana / OpenRouter), chunked + resumable uploads, bulk delete/reassign, idempotent retries. Capability discovery (F.7e) - Per-site credential probe + adapters for WordPress / WooCommerce / Gitea, tier-fit unions granted ∪ roles, capability badge UI with HTMX partial re-check, install hint in every companion-unreachable error. Companion plugin overhaul - Renamed wordpress-plugin/airano-mcp-seo-bridge → wordpress-plugin/airano-mcp-bridge. - Eight new endpoints: /capabilities, /bulk-meta, /export, /cache-purge, /transient-flush, /site-health, /audit-hook, /upload-and-attach. - wp.org Plugin Check pass: i18n, WP_Filesystem, scheme allowlist on audit-hook URL. Other - Gitea ergonomics (F.17): batch files, tree, search, compare, releases, fork. - Opportunistic bcrypt upgrade for legacy SHA-256 admin keys (F.8). - n8n refactor: structured errors, capability probe, missing tools backfilled. - Idempotency-Key dedup for AI media upload retries; WP client fast-fails on unreachable sites. Docs - README + CLAUDE.md drop the fixed "633 tools" claim. The total grows with each release; per-plugin approximations + dashboard-surfaced counts replace it. - Tools/Tests badges removed in favour of "Plugins: 10". Deployment - PyPI mirror chain, optional BUILD_HTTP_PROXY, Alpine→Yandex apk mirror, Debian-slim Plan-B Dockerfile, mirror.gcr.io variant. CI - Black + Ruff clean on Python 3.12; pytest tests/ green. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
475 lines
16 KiB
Python
475 lines
16 KiB
Python
"""Chunked upload session store (F.5a.5).
|
|
|
|
Server-side buffering for large media uploads: SQLite metadata + disk spill.
|
|
Sessions are deterministic (same user+file metadata yields the same
|
|
session_id), enabling resumable uploads. Enforces per-user concurrency
|
|
quota, per-session byte cap, and a 1h TTL reaped by a background task.
|
|
|
|
Chunks are appended sequentially; out-of-order or duplicate indexes raise
|
|
a typed error. Optional full-payload sha256 (supplied at `start`) is
|
|
verified when the session is finalized.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
import uuid
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from core.database import Database, get_database
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# --- Config ----------------------------------------------------------------
|
|
|
|
DEFAULT_SPILL_DIR = Path(os.environ.get("MCPHUB_UPLOAD_SPILL_DIR", "/tmp/mcphub-uploads"))
|
|
SESSION_TTL = timedelta(seconds=int(os.environ.get("MCPHUB_UPLOAD_TTL_SEC", "3600")))
|
|
MAX_SESSION_BYTES = int(os.environ.get("MCPHUB_UPLOAD_MAX_BYTES", str(500 * 1024 * 1024)))
|
|
MAX_CONCURRENT_PER_USER = int(os.environ.get("MCPHUB_UPLOAD_MAX_CONCURRENT", "10"))
|
|
|
|
|
|
# --- Errors ----------------------------------------------------------------
|
|
|
|
|
|
class UploadSessionError(Exception):
|
|
"""Typed session error with stable JSON code."""
|
|
|
|
def __init__(self, code: str, message: str, details: dict | None = None):
|
|
super().__init__(message)
|
|
self.code = code
|
|
self.message = message
|
|
self.details = details or {}
|
|
|
|
def to_dict(self) -> dict:
|
|
return {"error_code": self.code, "message": self.message, "details": self.details}
|
|
|
|
|
|
# --- Helpers ---------------------------------------------------------------
|
|
|
|
|
|
def _utc_now() -> datetime:
|
|
return datetime.now(UTC)
|
|
|
|
|
|
def _iso(dt: datetime) -> str:
|
|
return dt.isoformat()
|
|
|
|
|
|
def _parse_iso(value: str) -> datetime:
|
|
dt = datetime.fromisoformat(value)
|
|
if dt.tzinfo is None:
|
|
dt = dt.replace(tzinfo=UTC)
|
|
return dt
|
|
|
|
|
|
def make_session_id(
|
|
user_id: str,
|
|
filename: str,
|
|
total_bytes: int,
|
|
mime: str | None,
|
|
sha256: str | None,
|
|
) -> str:
|
|
"""Deterministic session id — same tuple → same id (enables resume)."""
|
|
payload = f"{user_id}|{filename}|{total_bytes}|{mime or ''}|{sha256 or ''}"
|
|
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
|
|
|
|
|
|
def _ensure_spill_dir(spill_dir: Path) -> None:
|
|
spill_dir.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
os.chmod(spill_dir, 0o700)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
# --- Store -----------------------------------------------------------------
|
|
|
|
|
|
class UploadSession:
|
|
"""Runtime view of an upload session row."""
|
|
|
|
__slots__ = (
|
|
"id",
|
|
"user_id",
|
|
"filename",
|
|
"total_bytes",
|
|
"mime",
|
|
"sha256",
|
|
"received_bytes",
|
|
"next_chunk",
|
|
"spill_path",
|
|
"status",
|
|
"created_at",
|
|
"expires_at",
|
|
)
|
|
|
|
def __init__(self, **kwargs: Any) -> None:
|
|
for name in self.__slots__:
|
|
setattr(self, name, kwargs.get(name))
|
|
|
|
@classmethod
|
|
def from_row(cls, row: dict[str, Any]) -> UploadSession:
|
|
return cls(
|
|
id=row["id"],
|
|
user_id=row["user_id"],
|
|
filename=row["filename"],
|
|
total_bytes=int(row["total_bytes"]),
|
|
mime=row["mime"],
|
|
sha256=row["sha256"],
|
|
received_bytes=int(row["received_bytes"]),
|
|
next_chunk=int(row["next_chunk"]),
|
|
spill_path=Path(row["spill_path"]),
|
|
status=row["status"],
|
|
created_at=_parse_iso(row["created_at"]),
|
|
expires_at=_parse_iso(row["expires_at"]),
|
|
)
|
|
|
|
def to_public_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"session_id": self.id,
|
|
"filename": self.filename,
|
|
"total_bytes": self.total_bytes,
|
|
"received_bytes": self.received_bytes,
|
|
"next_chunk": self.next_chunk,
|
|
"status": self.status,
|
|
"expires_at": _iso(self.expires_at),
|
|
}
|
|
|
|
|
|
class UploadSessionStore:
|
|
"""Persistent chunk-upload session store with disk spill."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
db: Database | None = None,
|
|
spill_dir: Path | None = None,
|
|
ttl: timedelta = SESSION_TTL,
|
|
max_session_bytes: int = MAX_SESSION_BYTES,
|
|
max_concurrent_per_user: int = MAX_CONCURRENT_PER_USER,
|
|
) -> None:
|
|
self._db = db
|
|
self.spill_dir = Path(spill_dir or DEFAULT_SPILL_DIR)
|
|
self.ttl = ttl
|
|
self.max_session_bytes = max_session_bytes
|
|
self.max_concurrent_per_user = max_concurrent_per_user
|
|
self._lock = asyncio.Lock()
|
|
_ensure_spill_dir(self.spill_dir)
|
|
|
|
@property
|
|
def db(self) -> Database:
|
|
return self._db or get_database()
|
|
|
|
# -- start -------------------------------------------------------------
|
|
|
|
async def start(
|
|
self,
|
|
*,
|
|
user_id: str,
|
|
filename: str,
|
|
total_bytes: int,
|
|
mime: str | None = None,
|
|
sha256: str | None = None,
|
|
) -> UploadSession:
|
|
if total_bytes <= 0:
|
|
raise UploadSessionError("BAD_SIZE", "total_bytes must be positive.")
|
|
if total_bytes > self.max_session_bytes:
|
|
raise UploadSessionError(
|
|
"SESSION_TOO_LARGE",
|
|
f"total_bytes {total_bytes} exceeds limit {self.max_session_bytes}.",
|
|
{"max": self.max_session_bytes},
|
|
)
|
|
|
|
session_id = make_session_id(user_id, filename, total_bytes, mime, sha256)
|
|
|
|
async with self._lock:
|
|
existing = await self._get_row(session_id)
|
|
if existing is not None:
|
|
sess = UploadSession.from_row(existing)
|
|
if sess.status == "open" and sess.expires_at > _utc_now():
|
|
return sess
|
|
# Stale/finished — replace
|
|
await self._delete_row(session_id)
|
|
_unlink_silent(sess.spill_path)
|
|
|
|
open_count = await self._count_open_for_user(user_id)
|
|
if open_count >= self.max_concurrent_per_user:
|
|
raise UploadSessionError(
|
|
"QUOTA_EXCEEDED",
|
|
f"User has {open_count} open upload sessions "
|
|
f"(max {self.max_concurrent_per_user}).",
|
|
{"open": open_count, "max": self.max_concurrent_per_user},
|
|
)
|
|
|
|
now = _utc_now()
|
|
expires = now + self.ttl
|
|
spill_path = self.spill_dir / f"{session_id}.part"
|
|
# Touch an empty spill file with 0600 perms
|
|
_ensure_spill_dir(self.spill_dir)
|
|
with open(spill_path, "wb") as f:
|
|
f.truncate(0)
|
|
try:
|
|
os.chmod(spill_path, 0o600)
|
|
except OSError:
|
|
pass
|
|
|
|
await self.db.execute(
|
|
"INSERT INTO upload_sessions "
|
|
"(id, user_id, filename, total_bytes, mime, sha256, "
|
|
" received_bytes, next_chunk, spill_path, status, "
|
|
" created_at, expires_at) "
|
|
"VALUES (?, ?, ?, ?, ?, ?, 0, 0, ?, 'open', ?, ?)",
|
|
(
|
|
session_id,
|
|
user_id,
|
|
filename,
|
|
total_bytes,
|
|
mime,
|
|
sha256,
|
|
str(spill_path),
|
|
_iso(now),
|
|
_iso(expires),
|
|
),
|
|
)
|
|
return UploadSession(
|
|
id=session_id,
|
|
user_id=user_id,
|
|
filename=filename,
|
|
total_bytes=total_bytes,
|
|
mime=mime,
|
|
sha256=sha256,
|
|
received_bytes=0,
|
|
next_chunk=0,
|
|
spill_path=spill_path,
|
|
status="open",
|
|
created_at=now,
|
|
expires_at=expires,
|
|
)
|
|
|
|
# -- append chunk ------------------------------------------------------
|
|
|
|
async def append_chunk(
|
|
self,
|
|
session_id: str,
|
|
index: int,
|
|
data: bytes,
|
|
*,
|
|
chunk_sha256: str | None = None,
|
|
) -> UploadSession:
|
|
async with self._lock:
|
|
sess = await self._require_open(session_id)
|
|
if index != sess.next_chunk:
|
|
raise UploadSessionError(
|
|
"CHUNK_ORDER",
|
|
f"Expected chunk index {sess.next_chunk}, got {index}.",
|
|
{"expected": sess.next_chunk, "got": index},
|
|
)
|
|
new_size = sess.received_bytes + len(data)
|
|
if new_size > sess.total_bytes:
|
|
raise UploadSessionError(
|
|
"CHUNK_OVERFLOW",
|
|
f"Chunk would exceed declared total_bytes "
|
|
f"({new_size} > {sess.total_bytes}).",
|
|
{"declared": sess.total_bytes, "would_be": new_size},
|
|
)
|
|
if chunk_sha256 is not None:
|
|
actual = hashlib.sha256(data).hexdigest()
|
|
if actual.lower() != chunk_sha256.lower():
|
|
raise UploadSessionError(
|
|
"CHUNK_CHECKSUM",
|
|
"Chunk sha256 does not match supplied value.",
|
|
{"expected": chunk_sha256, "actual": actual, "index": index},
|
|
)
|
|
|
|
with open(sess.spill_path, "ab") as f:
|
|
f.write(data)
|
|
sess.received_bytes = new_size
|
|
sess.next_chunk = index + 1
|
|
await self.db.execute(
|
|
"UPDATE upload_sessions SET received_bytes = ?, next_chunk = ? WHERE id = ?",
|
|
(sess.received_bytes, sess.next_chunk, session_id),
|
|
)
|
|
return sess
|
|
|
|
# -- finalize ----------------------------------------------------------
|
|
|
|
async def finalize(self, session_id: str) -> tuple[UploadSession, bytes]:
|
|
"""Read the full spill file and verify. Returns (session, bytes).
|
|
|
|
On checksum mismatch the session is kept (status remains 'open') so
|
|
the caller can retry; on success, the row and spill file are removed.
|
|
"""
|
|
async with self._lock:
|
|
sess = await self._require_open(session_id)
|
|
if sess.received_bytes != sess.total_bytes:
|
|
raise UploadSessionError(
|
|
"INCOMPLETE",
|
|
f"Received {sess.received_bytes}/{sess.total_bytes} bytes.",
|
|
{
|
|
"received": sess.received_bytes,
|
|
"total": sess.total_bytes,
|
|
},
|
|
)
|
|
with open(sess.spill_path, "rb") as f:
|
|
data = f.read()
|
|
if sess.sha256:
|
|
actual = hashlib.sha256(data).hexdigest()
|
|
if actual.lower() != sess.sha256.lower():
|
|
raise UploadSessionError(
|
|
"CHECKSUM_MISMATCH",
|
|
"Assembled sha256 does not match value supplied at start.",
|
|
{"expected": sess.sha256, "actual": actual},
|
|
)
|
|
# Success — drop the session
|
|
await self._delete_row(session_id)
|
|
_unlink_silent(sess.spill_path)
|
|
return sess, data
|
|
|
|
# -- abort -------------------------------------------------------------
|
|
|
|
async def abort(self, session_id: str) -> bool:
|
|
async with self._lock:
|
|
row = await self._get_row(session_id)
|
|
if row is None:
|
|
return False
|
|
sess = UploadSession.from_row(row)
|
|
await self._delete_row(session_id)
|
|
_unlink_silent(sess.spill_path)
|
|
return True
|
|
|
|
# -- get ---------------------------------------------------------------
|
|
|
|
async def get(self, session_id: str) -> UploadSession | None:
|
|
row = await self._get_row(session_id)
|
|
return UploadSession.from_row(row) if row else None
|
|
|
|
# -- cleanup -----------------------------------------------------------
|
|
|
|
async def cleanup_expired(self, *, now: datetime | None = None) -> int:
|
|
now = now or _utc_now()
|
|
rows = await self.db.fetchall(
|
|
"SELECT * FROM upload_sessions WHERE expires_at < ?", (_iso(now),)
|
|
)
|
|
reaped = 0
|
|
for row in rows:
|
|
sess = UploadSession.from_row(row)
|
|
_unlink_silent(sess.spill_path)
|
|
await self._delete_row(sess.id)
|
|
reaped += 1
|
|
if reaped:
|
|
logger.info("Reaped %d expired upload session(s)", reaped)
|
|
return reaped
|
|
|
|
# -- internals ---------------------------------------------------------
|
|
|
|
async def _get_row(self, session_id: str) -> dict[str, Any] | None:
|
|
return await self.db.fetchone("SELECT * FROM upload_sessions WHERE id = ?", (session_id,))
|
|
|
|
async def _delete_row(self, session_id: str) -> None:
|
|
await self.db.execute("DELETE FROM upload_sessions WHERE id = ?", (session_id,))
|
|
|
|
async def _count_open_for_user(self, user_id: str) -> int:
|
|
row = await self.db.fetchone(
|
|
"SELECT COUNT(*) AS c FROM upload_sessions "
|
|
"WHERE user_id = ? AND status = 'open' AND expires_at > ?",
|
|
(user_id, _iso(_utc_now())),
|
|
)
|
|
return int(row["c"]) if row else 0
|
|
|
|
async def _require_open(self, session_id: str) -> UploadSession:
|
|
row = await self._get_row(session_id)
|
|
if row is None:
|
|
raise UploadSessionError(
|
|
"NO_SESSION", f"Session {session_id} not found.", {"session_id": session_id}
|
|
)
|
|
sess = UploadSession.from_row(row)
|
|
if sess.status != "open":
|
|
raise UploadSessionError(
|
|
"BAD_STATE",
|
|
f"Session {session_id} is in state '{sess.status}'.",
|
|
{"state": sess.status},
|
|
)
|
|
if sess.expires_at <= _utc_now():
|
|
raise UploadSessionError(
|
|
"EXPIRED", f"Session {session_id} has expired.", {"session_id": session_id}
|
|
)
|
|
return sess
|
|
|
|
|
|
def _unlink_silent(path: Path) -> None:
|
|
try:
|
|
path.unlink(missing_ok=True)
|
|
except OSError as e:
|
|
logger.debug("Could not remove spill file %s: %s", path, e)
|
|
|
|
|
|
# --- Singleton + background cleanup ---------------------------------------
|
|
|
|
_store: UploadSessionStore | None = None
|
|
|
|
|
|
def get_upload_session_store() -> UploadSessionStore:
|
|
global _store
|
|
if _store is None:
|
|
_store = UploadSessionStore()
|
|
return _store
|
|
|
|
|
|
def set_upload_session_store(store: UploadSessionStore | None) -> None:
|
|
"""Override the singleton (used by tests)."""
|
|
global _store
|
|
_store = store
|
|
|
|
|
|
class CleanupTask:
|
|
"""Periodically reaps expired sessions. Register in server lifespan."""
|
|
|
|
def __init__(
|
|
self,
|
|
store: UploadSessionStore | None = None,
|
|
interval_seconds: int = 300,
|
|
) -> None:
|
|
self._store = store
|
|
self.interval = interval_seconds
|
|
self._task: asyncio.Task | None = None
|
|
self._stop = asyncio.Event()
|
|
self.id = uuid.uuid4().hex[:8]
|
|
|
|
@property
|
|
def store(self) -> UploadSessionStore:
|
|
return self._store or get_upload_session_store()
|
|
|
|
async def start(self) -> None:
|
|
if self._task is not None:
|
|
return
|
|
self._stop.clear()
|
|
self._task = asyncio.create_task(self._run(), name=f"upload-cleanup-{self.id}")
|
|
|
|
async def stop(self) -> None:
|
|
if self._task is None:
|
|
return
|
|
self._stop.set()
|
|
self._task.cancel()
|
|
try:
|
|
await self._task
|
|
except (asyncio.CancelledError, Exception):
|
|
pass
|
|
self._task = None
|
|
|
|
async def _run(self) -> None:
|
|
while not self._stop.is_set():
|
|
try:
|
|
await self.store.cleanup_expired()
|
|
except Exception as e: # noqa: BLE001
|
|
logger.warning("Upload-session cleanup error: %s", e)
|
|
try:
|
|
await asyncio.wait_for(self._stop.wait(), timeout=self.interval)
|
|
except TimeoutError:
|
|
continue
|