commit c94214d9de0e27ea6099e68500ed54a0accb9540
equwal <truex@equwal.com>
2026-09-22 15:23:42 -0700
Keep every upload and result on the server
The owner debugs conversions from their files and deletes the files by
hand. So the server now deletes no file by itself:
- A server job keeps its audio, its book and its subtitles together in
data/work/<job id>/ after it succeeds or is cancelled.
- The hourly housekeeping no longer deletes old uploads, drafts or
results. It only lets go of abandoned browser jobs. backend/retention.py
and its two settings are gone.
- A refused upload keeps what arrived.
- DELETE /api/jobs/{id} takes the job off the visitor's list and keeps
its files.
The terms and the note for server conversions now say that the server
keeps the files, that we look at them only to find faults, and that we
delete them on request.
.env.example | 3 --
README.md | 17 ++++----
backend/api.py | 12 ++----
backend/main.py | 22 +++++-----
backend/retention.py | 84 --------------------------------------
backend/runner.py | 19 +--------
backend/settings.py | 4 --
frontend/app.js | 3 +-
frontend/terms.html | 17 +++-----
tests/test_keep_files.py | 102 +++++++++++++++++++++++++++++++++++++++++++++++
tests/test_retention.py | 80 -------------------------------------
11 files changed, 138 insertions(+), 225 deletions(-)
diff --git a/.env.example b/.env.example
index ab3b5d2..28fd5f1 100644
--- a/.env.example
+++ b/.env.example
@@ -62,9 +62,6 @@ SUBPLZ_WEB_FREE_CREDITS_ANONYMOUS=1
SUBPLZ_WEB_FREE_CREDITS_SIGNED_IN=10
# Where customers write to. Shown on /terms.html.
SUBPLZ_WEB_CONTACT_EMAIL=
-# How long the server keeps a visitor's files (see backend/retention.py).
-SUBPLZ_WEB_INPUT_RETENTION_HOURS=24
-SUBPLZ_WEB_ARTIFACT_RETENTION_DAYS=7
# --- accounts & payments ---------------------------------------------------
# The origin users reach this on. Sign-in links and Stripe return URLs use it.
diff --git a/README.md b/README.md
index 77c19c7..a4b60b9 100644
--- a/README.md
+++ b/README.md
@@ -263,8 +263,10 @@ m4a/m4b, which are the types an EPUB 3 reader must play. `tests/engine/epub.test
follows each overlay as a reader does, and gives the result to the W3C
`epubcheck` when `EPUBCHECK` points to its jar.
-Uploaded media is deleted once a job succeeds. A **failed** job keeps its inputs
-so you can fix the language and retry without re-uploading.
+The server keeps the files of each job together in `data/work/<job id>/`: the
+audio and the book in `input/`, the subtitles in `out/`, and the run log. Nothing
+deletes them: the operator deletes them by hand. A **failed** job can thus be
+retried without a new upload.
---
@@ -354,11 +356,9 @@ Off: the files are uploaded, the job runs on the server, and the result waits
in the list of conversions, so the visitor can close the tab. The page asks
for the credit before the upload, not after it.
-The server does not keep a visitor's files past their use
-(`backend/retention.py`, once an hour): uploads go when the job succeeds, or
-after 24 hours when it did not; results go after 7 days. `/terms.html` says the
-same to the visitor, with the numbers read from the server, and shows
-`SUBPLZ_WEB_CONTACT_EMAIL`.
+The server keeps each upload and each result, for debugging. Nothing deletes
+them automatically: the operator deletes them by hand, and on request.
+`/terms.html` says so to the visitor, and shows `SUBPLZ_WEB_CONTACT_EMAIL`.
`SUBPLZ_WEB_CLOUD_ENABLED` says that fast conversion is on offer. While it is
false the page shows no way to buy credits, whatever `SUBPLZ_WEB_BILLING_ENABLED`
@@ -440,7 +440,8 @@ shared filesystem.
- Set the Stripe keys, the webhook and SMTP (see *Turning payments on*).
- Put a reverse proxy in front for TLS and upload limits.
-- Add a retention job — audiobooks are large and artifacts are kept forever.
+- Watch the disk. The server keeps each upload and result until the operator
+ deletes it.
- Run workers on hardware that can take it. Alignment needs roughly 2–3 GB of
RAM; a 1 GB VPS will OOM. A 4h40m Russian audiobook took ~45 minutes on
`tiny`/CPU with 15 threads here, and would take many hours on one vCPU.
diff --git a/backend/api.py b/backend/api.py
index 8d31b72..f5f1871 100644
--- a/backend/api.py
+++ b/backend/api.py
@@ -13,7 +13,6 @@ checkout, its return trip and its webhook).
from __future__ import annotations
-import shutil
from pathlib import Path
from typing import Annotated, Literal
@@ -324,8 +323,6 @@ def get_pricing():
"payments_available": settings.payments_configured,
"tiers": pricing.TIER_OUTPUTS,
"contact_email": settings.contact_email,
- "input_retention_hours": settings.input_retention_hours,
- "artifact_retention_days": settings.artifact_retention_days,
"plans": pricing.as_dicts(),
}
@@ -573,14 +570,13 @@ async def create_upload(
session.add(job)
session.commit()
+ # A refused upload keeps what arrived, for debugging. The operator deletes
+ # it by hand.
except HTTPException:
- shutil.rmtree(paths.root, ignore_errors=True)
raise
except detect.DetectionError as exc:
- shutil.rmtree(paths.root, ignore_errors=True)
raise HTTPException(400, str(exc)) from exc
except Exception as exc: # noqa: BLE001
- shutil.rmtree(paths.root, ignore_errors=True)
raise HTTPException(500, f"Upload failed: {exc}") from exc
return UploadOut(
@@ -857,11 +853,11 @@ def delete_job(
account: Annotated[Account, Depends(get_account)],
session: Annotated[Session, Depends(get_session)],
):
+ """Take the job off the visitor's list. Its files stay on the server for
+ debugging, and the operator deletes them by hand."""
job = _load(session, account, job_id)
if job.status in (JobStatus.queued, JobStatus.running):
raise HTTPException(409, "Cancel the job before deleting it.")
- storage.delete_prefix(job.id)
- shutil.rmtree(Paths.for_job(job.id).root, ignore_errors=True)
session.delete(job)
session.commit()
return {"deleted": job_id}
diff --git a/backend/main.py b/backend/main.py
index a6ee76e..c08cf4b 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -97,19 +97,23 @@ async def lifespan(_: FastAPI):
queue.shutdown()
-async def _housekeeping() -> None:
- """Once an hour: let go of abandoned browser jobs, and delete files past their time."""
- from . import retention
+def housekeeping() -> None:
+ """One pass of the hourly housekeeping: let go of abandoned browser jobs.
+
+ It deletes no file. The server keeps each upload and each result for
+ debugging, and the operator deletes them by hand.
+ """
from .api import expire_stale_local_jobs
- while True:
- def once() -> None:
- with SessionLocal() as s:
- expire_stale_local_jobs(s)
- retention.sweep(s)
+ with SessionLocal() as s:
+ expire_stale_local_jobs(s)
+
+async def _housekeeping() -> None:
+ """Run `housekeeping` once an hour."""
+ while True:
try:
- await asyncio.to_thread(once) # deleting a book's files is slow; do not hold the site up
+ await asyncio.to_thread(housekeeping)
except Exception: # noqa: BLE001 - housekeeping must not take the site down
log.exception("housekeeping failed")
await asyncio.sleep(3600)
diff --git a/backend/retention.py b/backend/retention.py
deleted file mode 100644
index e2296dc..0000000
--- a/backend/retention.py
+++ /dev/null
@@ -1,84 +0,0 @@
-"""What the server keeps, and for how long.
-
-A server job brings an audiobook and a book onto this machine. They are the
-visitor's files, often bought ones, and the disk is small. So nothing stays
-longer than it is of use:
-
- uploaded files deleted when the job succeeds (runner.py). A job that was
- never started, failed or was cancelled keeps them for
- `input_retention_hours`, so the visitor can fix the language
- and try again; then they go too.
- results subtitles and videos stay for `artifact_retention_days`,
- then they are deleted. The row of the job stays, so the list
- still says what was converted.
-
-Jobs that ran in the visitor's browser have no files here, only the subtitles
-the browser sent back; those follow the rule for results.
-"""
-
-from __future__ import annotations
-
-import logging
-import shutil
-from datetime import timedelta
-
-from sqlalchemy.orm import Session
-
-from .db import Artifact, Job, JobStatus, utcnow
-from .runner import Paths
-from .settings import settings
-from .storage import storage
-
-log = logging.getLogger("subplz-web.retention")
-
-_IDLE = [JobStatus.draft, JobStatus.failed, JobStatus.canceled]
-
-
-def sweep(session: Session) -> dict[str, int]:
- """Delete what is past its time. Safe to run at any moment, and again."""
- now = utcnow()
- done = {"inputs": 0, "drafts": 0, "results": 0}
-
- old_idle = (
- session.query(Job)
- .filter(Job.local == 0, Job.status.in_(_IDLE),
- Job.created_at < now - timedelta(hours=settings.input_retention_hours))
- .all()
- )
- for job in old_idle:
- root = Paths.for_job(job.id).root
- if root.exists():
- shutil.rmtree(root, ignore_errors=True)
- done["inputs"] += 1
- _delete_stored(job.id)
- if job.status == JobStatus.draft:
- # Never started: nothing to show for it.
- session.delete(job)
- done["drafts"] += 1
-
- old_results = (
- session.query(Job)
- .join(Artifact, Artifact.job_id == Job.id)
- .filter(Job.status == JobStatus.succeeded,
- Job.finished_at < now - timedelta(days=settings.artifact_retention_days))
- .distinct()
- .all()
- )
- for job in old_results:
- _delete_stored(job.id)
- shutil.rmtree(Paths.for_job(job.id).root, ignore_errors=True)
- session.query(Artifact).filter(Artifact.job_id == job.id).delete()
- job.stage = f"Files deleted after {settings.artifact_retention_days} days"
- done["results"] += 1
-
- session.commit()
- if any(done.values()):
- log.info("retention sweep: %s", done)
- return done
-
-
-def _delete_stored(job_id: str) -> None:
- try:
- storage.delete_prefix(job_id)
- except Exception as exc: # noqa: BLE001 - the next sweep tries again
- log.warning("could not delete stored files of %s: %s", job_id, exc)
diff --git a/backend/runner.py b/backend/runner.py
index 58430f6..327bae1 100644
--- a/backend/runner.py
+++ b/backend/runner.py
@@ -380,7 +380,6 @@ def run_job(job_id: str) -> None:
returncode = _stream_aligner(job_id, request, log_path)
if _is_canceled(job_id):
- _cleanup_inputs(paths)
return
if returncode != 0:
@@ -392,15 +391,13 @@ def run_job(job_id: str) -> None:
_set(job_id, stage="Collecting output", progress=0.92)
_collect_artifacts(job_id, paths, log_path, duration, request, audio_in)
+ # The audio, the book and the subtitles stay in the folder of the job
+ # for debugging. The operator deletes them by hand.
_set(job_id, status=JobStatus.succeeded, stage="Done", progress=1.0,
finished_at=utcnow())
- # Only on success: a failed job keeps its inputs so it can be retried
- # without re-uploading hundreds of megabytes.
- _cleanup_inputs(paths)
except Exception as exc: # noqa: BLE001 - surfaced to the user on the Job row
if _is_canceled(job_id):
- _cleanup_inputs(paths)
return
# Keep the log even on failure: it is the only way to debug an alignment.
try:
@@ -671,15 +668,3 @@ def _summarize_srt(path: Path) -> tuple[int, float | None, float | None]:
if not stamps:
return count, None, None
return count, to_seconds(stamps[0]), to_seconds(stamps[-1])
-
-
-def _cleanup_inputs(paths: Paths) -> None:
- """Drop the uploaded media once the run is over; keep artifacts and the log."""
- shutil.rmtree(paths.inp, ignore_errors=True)
- shutil.rmtree(paths.out, ignore_errors=True)
- shutil.rmtree(paths.root / "video", ignore_errors=True)
- # Also drop the shared-storage copy made for external workers.
- try:
- storage.delete_prefix(f"{paths.root.name}/input")
- except Exception:
- pass
diff --git a/backend/settings.py b/backend/settings.py
index d41129f..9b5de23 100644
--- a/backend/settings.py
+++ b/backend/settings.py
@@ -54,10 +54,6 @@ class Settings(BaseSettings):
# Presigned-URL lifetime for s3 downloads.
download_url_ttl_seconds: int = 3600
- # How long the server keeps a visitor's files. See retention.py.
- input_retention_hours: int = 24
- artifact_retention_days: int = 7
-
# SQLite locally; set to a postgresql+psycopg:// URL in production.
database_url: str = ""
diff --git a/frontend/app.js b/frontend/app.js
index 2a0c9ab..59bd27b 100644
--- a/frontend/app.js
+++ b/frontend/app.js
@@ -386,7 +386,8 @@ function renderMode() {
el.start.textContent = 'Upload and convert on our server';
el.eta.textContent =
'The files are uploaded, the work is done on our server, and the result waits for you on this page, ' +
- 'so you can close this tab. The files are deleted after the job.' + serverPrice();
+ 'so you can close this tab. We keep the files on our server to find faults, and delete them on request.' +
+ serverPrice();
}
el.inBrowser.addEventListener('change', renderMode);
diff --git a/frontend/terms.html b/frontend/terms.html
index a6054f7..0f1874b 100644
--- a/frontend/terms.html
+++ b/frontend/terms.html
@@ -31,13 +31,11 @@
<h2>Your files</h2>
<ul>
<li>Use only files that you have the right to use. You keep all rights in them. We claim none.</li>
- <li>Files uploaded for a server conversion are deleted when the job succeeds. If a job fails,
- is cancelled, or is never started, they are kept for <span class="input-hours">24</span> hours
- so that you can try again, and then deleted.</li>
- <li>Results (subtitles, videos) are kept for <span class="result-days">7</span> days and then
- deleted. Download what you want to keep.</li>
- <li>Nobody looks at your files, they are not shared, and they are not used to train anything.
- There is no public library of uploads.</li>
+ <li>The server keeps the files that you upload for a server conversion (the audio and the
+ book), and the results of each conversion (subtitles, videos). Nothing deletes them
+ automatically: we delete them by hand. Write to us to have your files deleted.</li>
+ <li>We look at your files only to find and fix faults in a conversion. They are not shared,
+ and they are not used to train anything. There is no public library of uploads.</li>
</ul>
<h2>Payments and refunds</h2>
@@ -66,7 +64,7 @@
</ul>
<h2>Copyright complaints</h2>
- <p>We do not publish or share what users upload, and uploads are deleted as said above. If you
+ <p>We do not publish or share what users upload. If you
believe that material on this service infringes your copyright, write to the contact address
with: the work, where it is on this service, your contact details, a statement that you act in
good faith for the owner of the right, and your signature. We remove what the notice properly
@@ -83,7 +81,6 @@
</main>
<script type="module">
const site = await (await fetch('/api/pricing')).json();
- const put = (cls, value) => document.querySelectorAll('.' + cls).forEach((el) => { el.textContent = value; });
if (site.contact_email) {
document.querySelectorAll('.contact').forEach((el) => {
const a = document.createElement('a');
@@ -92,8 +89,6 @@
el.replaceChildren(a);
});
}
- put('input-hours', site.input_retention_hours);
- put('result-days', site.artifact_retention_days);
</script>
</body>
</html>
diff --git a/tests/test_keep_files.py b/tests/test_keep_files.py
new file mode 100644
index 0000000..4f42d25
--- /dev/null
+++ b/tests/test_keep_files.py
@@ -0,0 +1,102 @@
+"""The server keeps each file of a job: the audio, the book and the subtitles,
+together in the folder of the job. Nothing deletes them automatically. The
+operator deletes them by hand."""
+
+from __future__ import annotations
+
+from datetime import timedelta
+
+from backend import main, runner
+from backend.db import Job, JobStatus, SessionLocal, utcnow
+from backend.runner import Paths
+from backend.settings import settings
+
+from .conftest import get_job_row, make_job
+
+SRT = "1\n00:00:00,000 --> 00:00:01,000\nhello\n\n"
+
+
+def stage_inputs(job_id: str) -> tuple[Paths, list]:
+ """Put an audio file and a book where an upload puts them."""
+ paths = Paths.for_job(job_id)
+ paths.create()
+ audio = runner.staged_audio_path(job_id, "book.m4b")
+ book = runner.staged_text_path(job_id, "book.epub")
+ audio.write_bytes(b"audio")
+ book.write_bytes(b"book")
+ return paths, [audio, book]
+
+
+def age(job_id: str, **delta) -> None:
+ with SessionLocal() as s:
+ job = s.get(Job, job_id)
+ job.created_at = utcnow() - timedelta(**delta)
+ job.finished_at = utcnow() - timedelta(**delta)
+ s.commit()
+
+
+def test_a_finished_server_job_keeps_audio_book_and_subtitles_together(client, monkeypatch):
+ job_id = make_job(client, JobStatus.queued)
+ paths, inputs = stage_inputs(job_id)
+ produced = paths.out / "source.en.srt"
+
+ def fake_aligner(job_id, request, log_path):
+ # Bytes, not text: on Windows write_text writes CRLF line ends.
+ produced.write_bytes(SRT.encode("utf-8"))
+ return 0
+
+ monkeypatch.setattr(runner, "probe_duration", lambda path: 1.0)
+ monkeypatch.setattr(runner, "probe_chapters", lambda path: 0)
+ monkeypatch.setattr(runner, "normalize_for_alignment", lambda job_id, source: source)
+ monkeypatch.setattr(runner, "_stream_aligner", fake_aligner)
+ monkeypatch.setattr(runner.aligner, "locate_output", lambda request: produced)
+ monkeypatch.setattr(settings, "render_video", False)
+
+ runner.run_job(job_id)
+
+ assert get_job_row(job_id).status == JobStatus.succeeded
+ for path in [*inputs, produced]:
+ assert path.exists(), path
+ assert paths.root in path.parents # one folder for the job
+ assert client.get(f"/api/jobs/{job_id}/files/srt").text == SRT
+
+
+def test_housekeeping_deletes_no_file_and_no_job(client):
+ failed = make_job(client, JobStatus.failed)
+ draft = make_job(client)
+ done = make_job(client, JobStatus.succeeded, with_files=True)
+ staged = {job_id: stage_inputs(job_id)[1] for job_id in (failed, draft, done)}
+ for job_id in staged:
+ age(job_id, days=400)
+
+ main.housekeeping()
+
+ for job_id, inputs in staged.items():
+ assert get_job_row(job_id) is not None
+ assert all(path.exists() for path in inputs), job_id
+ assert client.get(f"/api/jobs/{done}/files/srt").status_code == 200
+
+
+def test_deleting_a_job_keeps_its_files(client):
+ job_id = make_job(client)
+ _, inputs = stage_inputs(job_id)
+ assert client.delete(f"/api/jobs/{job_id}").status_code == 200
+ assert get_job_row(job_id) is None
+ assert all(path.exists() for path in inputs)
+
+
+def test_a_refused_upload_keeps_what_arrived(client, monkeypatch):
+ monkeypatch.setattr(settings, "max_upload_bytes", 10)
+ work = settings.data_dir / "work"
+ before = set(work.iterdir()) if work.exists() else set()
+
+ r = client.post("/api/uploads", files=[
+ ("files", ("book.epub", b"epub", "application/epub+zip")),
+ ("files", ("book.m4b", b"x" * 100, "audio/mp4")),
+ ])
+ assert r.status_code == 413
+
+ new = set(work.iterdir()) - before
+ assert len(new) == 1
+ kept = {p.suffix for p in new.pop().rglob("*") if p.is_file()}
+ assert kept == {".epub", ".m4b"}
diff --git a/tests/test_retention.py b/tests/test_retention.py
deleted file mode 100644
index 2086ce0..0000000
--- a/tests/test_retention.py
+++ /dev/null
@@ -1,80 +0,0 @@
-"""Uploaded books and results do not stay on the server past their time."""
-
-from __future__ import annotations
-
-from datetime import timedelta
-
-from backend import retention
-from backend.db import Artifact, Job, JobStatus, SessionLocal, utcnow
-from backend.runner import Paths
-from backend.storage import storage
-
-from .conftest import get_job_row, make_job
-
-
-def age(job_id: str, **delta) -> None:
- with SessionLocal() as s:
- job = s.get(Job, job_id)
- job.created_at = utcnow() - timedelta(**delta)
- job.finished_at = utcnow() - timedelta(**delta)
- s.commit()
-
-
-def stage_inputs(job_id: str):
- paths = Paths.for_job(job_id)
- paths.create()
- (paths.inp / "book.m4b").write_bytes(b"audio")
- return paths
-
-
-def sweep() -> dict:
- with SessionLocal() as s:
- return retention.sweep(s)
-
-
-def test_a_failed_job_keeps_its_upload_for_a_day_and_no_longer(client):
- job_id = make_job(client, JobStatus.failed)
- paths = stage_inputs(job_id)
-
- age(job_id, hours=23)
- sweep()
- assert (paths.inp / "book.m4b").exists() # still there for another try
-
- age(job_id, hours=25)
- assert sweep()["inputs"] == 1
- assert not paths.root.exists()
- assert get_job_row(job_id).status == JobStatus.failed # the list still shows it
-
-
-def test_an_upload_that_was_never_started_goes_away_with_its_job(client):
- job_id = make_job(client) # a draft
- paths = stage_inputs(job_id)
- age(job_id, hours=25)
- assert sweep()["drafts"] == 1
- assert not paths.root.exists() and get_job_row(job_id) is None
-
-
-def test_results_are_deleted_after_a_week_and_the_job_says_so(client):
- fresh = make_job(client, JobStatus.succeeded, with_files=True)
- old = make_job(client, JobStatus.succeeded, with_files=True)
- age(fresh, days=6)
- age(old, days=8)
-
- assert sweep()["results"] == 1
- assert client.get(f"/api/jobs/{fresh}/files/srt").status_code == 200
- assert client.get(f"/api/jobs/{old}/files/srt").status_code == 404
- job = client.get(f"/api/jobs/{old}").json()
- assert job["artifacts"] == [] and "deleted after 7 days" in job["stage"]
- with SessionLocal() as s:
- assert s.query(Artifact).filter(Artifact.job_id == old).count() == 0
- assert not storage.exists(f"{old}/book.en.srt")
-
- assert sweep() == {"inputs": 0, "drafts": 0, "results": 0} # and once is enough
-
-
-def test_running_jobs_are_left_alone(client):
- job_id = make_job(client, JobStatus.running)
- paths = stage_inputs(job_id)
- age(job_id, days=30)
- sweep()
- assert (paths.inp / "book.m4b").exists()