diff --git a/.obsidian/plugins/hoarder-sync/data.json b/.obsidian/plugins/hoarder-sync/data.json index 683b291..fef4a42 100644 --- a/.obsidian/plugins/hoarder-sync/data.json +++ b/.obsidian/plugins/hoarder-sync/data.json @@ -4,7 +4,7 @@ "syncFolder": "Hoarder", "attachmentsFolder": "Hoarder/attachments", "syncIntervalMinutes": 60, - "lastSyncTimestamp": 1777492621168, + "lastSyncTimestamp": 1777496221546, "updateExistingFiles": false, "excludeArchived": true, "onlyFavorites": false, diff --git a/Hoarder/2026-04-29-Фермерская-жизнь-в-ином-мире-2.md b/Hoarder/2026-04-29-Фермерская-жизнь-в-ином-мире-2.md new file mode 100644 index 0000000..4bc102d --- /dev/null +++ b/Hoarder/2026-04-29-Фермерская-жизнь-в-ином-мире-2.md @@ -0,0 +1,35 @@ +--- +bookmark_id: "gknfa711l5vpiogq1gbh1pdz" +url: | + https://v23.astar.bz/10838-fermerskaya-zhizn-v-inom-mire-2-sezon-isekai-nonbiri-nouka-2.html +title: | + Фермерская жизнь в ином мире 2 сезон , Isekai Nonbiri Nouka 2 смотреть онлайн или скачать бесплатно +date: 2026-04-29T20:17:43.000Z +modified: 2026-04-29T20:18:01.000Z +tags: + - Isekai + - Fantasy + - Anime + - Adventure + - Slice-Of-Life +note: +original_note: +summary: +banner: "[[Hoarder/attachments/924ae99c-fd3b-483f-a288-adee63415766-Фермерская-жизнь-в-ином-мире.jpg]]" +screenshot: "[[Hoarder/attachments/1de8d684-8ce7-4191-809e-1557a73b10ec-Фермерская-жизнь-в-ином-мире.jpg]]" + +--- + +# Фермерская жизнь в ином мире 2 сезон , Isekai Nonbiri Nouka 2 смотреть онлайн или скачать бесплатно + +![Фермерская жизнь в ином мире 2 сезон , Isekai Nonbiri Nouka 2 смотреть онлайн или скачать бесплатно - Banner Image](Hoarder/attachments/924ae99c-fd3b-483f-a288-adee63415766-Фермерская-жизнь-в-ином-мире.jpg) + +![Фермерская жизнь в ином мире 2 сезон , Isekai Nonbiri Nouka 2 смотреть онлайн или скачать бесплатно - Screenshot](Hoarder/attachments/1de8d684-8ce7-4191-809e-1557a73b10ec-Фермерская-жизнь-в-ином-мире.jpg) + +## Notes + + + +[Visit Link](https://v23.astar.bz/10838-fermerskaya-zhizn-v-inom-mire-2-sezon-isekai-nonbiri-nouka-2.html) + +[View in Hoarder](https://links.ai-impress.com/dashboard/preview/gknfa711l5vpiogq1gbh1pdz) \ No newline at end of file diff --git a/Hoarder/attachments/1de8d684-8ce7-4191-809e-1557a73b10ec-Фермерская-жизнь-в-ином-мире.jpg b/Hoarder/attachments/1de8d684-8ce7-4191-809e-1557a73b10ec-Фермерская-жизнь-в-ином-мире.jpg new file mode 100644 index 0000000..6572a23 Binary files /dev/null and b/Hoarder/attachments/1de8d684-8ce7-4191-809e-1557a73b10ec-Фермерская-жизнь-в-ином-мире.jpg differ diff --git a/Hoarder/attachments/924ae99c-fd3b-483f-a288-adee63415766-Фермерская-жизнь-в-ином-мире.jpg b/Hoarder/attachments/924ae99c-fd3b-483f-a288-adee63415766-Фермерская-жизнь-в-ином-мире.jpg new file mode 100644 index 0000000..a2f77a2 Binary files /dev/null and b/Hoarder/attachments/924ae99c-fd3b-483f-a288-adee63415766-Фермерская-жизнь-в-ином-мире.jpg differ diff --git a/wiki/_master-index.md b/wiki/_master-index.md index 54d91e0..7849e3e 100644 --- a/wiki/_master-index.md +++ b/wiki/_master-index.md @@ -21,9 +21,9 @@ This 3-hop pattern works for hundreds of articles without vector search. | [[wiki/obsidian-rag/_index\|obsidian-rag/]] | Karpathy's LLM wiki method — Obsidian RAG, setup, vs true RAG | 3 | | [[wiki/projects-overview/_index\|projects-overview/]] | All 42 Oliver Agency projects — grouped by server (optical-web-1, optical-dev, baic, box-cli) | 1 | | [[wiki/tech-patterns/_index\|tech-patterns/]] | Recurring tech stacks: FastAPI, React/Vite, Next.js, Azure AD, AI, Box, One2Edit, Redis/Celery, cost-tracker | 15 | -| [[wiki/architecture/_index\|architecture/]] | Cross-cutting architectural patterns: Docker Compose, multi-agent AI, GCP timeout, RAG, hotfolder, optical-dev deploy, cost-tracker, new-project checklist, troubleshooting playbooks, ADR log | 10 | +| [[wiki/architecture/_index\|architecture/]] | Cross-cutting architectural patterns: Docker Compose, multi-agent AI, GCP timeout, RAG, hotfolder, optical-dev deploy, cost-tracker, new-project checklist, troubleshooting playbooks, ADR log, Cloud Run Jobs | 11 | | [[wiki/client-knowledge/_index\|client-knowledge/]] | Per-client notes for Ford, H&M, L'Oréal, Barclays, Ferrero, 3M | 6 | -| [[wiki/concepts/_index\|concepts/]] | Atomic knowledge extracted from Claude Code sessions | 69 | +| [[wiki/concepts/_index\|concepts/]] | Atomic knowledge extracted from Claude Code sessions | 73 | | [[wiki/connections/_index\|connections/]] | Cross-cutting insights linking 2+ concepts: FastAPI+Azure AD+Docker trinity, AI→cost-tracker, Apache+Vite basePath, GCP→REST polling, Box+hotfolder, Docker DNS+AdGuard | 9 | | [[wiki/qa/_index\|qa/]] | Filed answers to queries (saved with `--file-back`) | 0 | | [[wiki/homelab/_index\|homelab/]] | Self-hosted infra: Proxmox install, IOMMU/PCI passthrough, hypervisor setup, budget builds, HP Elitedesk G3, Homarr API + Apps + Boards + Certificates + Integrations + Settings + Tasks + AdGuard + Clock + Docker Stats + Docker Integration + Download Client + Firewall + Proxmox Integration + Radarr + Readarr + Sonarr + Bookmarks + Calendar + Icons + App Widget + Weather + GitHub + Nextcloud + qBittorrent + RSS Feed + Speedtest Tracker + System Health Monitoring + System Resources + Services Map + Media Stack | 39 | diff --git a/wiki/architecture/_index.md b/wiki/architecture/_index.md index d4e2bc4..1a7ea78 100644 --- a/wiki/architecture/_index.md +++ b/wiki/architecture/_index.md @@ -3,7 +3,7 @@ title: "Architecture Patterns Index" description: "Cross-cutting architectural decisions across Oliver Agency projects" tags: [index, architecture] created: 2026-04-15 -updated: 2026-04-27 +updated: 2026-04-29 --- # Architecture Patterns @@ -24,6 +24,7 @@ Cross-cutting architectural decisions that appear in multiple Oliver projects. | [[wiki/architecture/new-project-checklist\|new-project-checklist]] | Step-by-step Oliver project setup — repo, Docker Compose, Azure AD, cost tracker, optical-dev deploy | All new projects | | [[wiki/architecture/troubleshooting-playbooks\|troubleshooting-playbooks]] | Failure → diagnosis → fix for FastAPI, Docker, React/Vite, Azure AD, Apache, PostgreSQL | All Oliver projects | | [[wiki/architecture/adr-log\|adr-log]] | Architecture Decision Records — why HTTP polling, Docker Compose, FastAPI, Azure AD, cost tracker were chosen | All Oliver projects | +| [[wiki/architecture/cloud-run-jobs-celery\|cloud-run-jobs-celery]] | Moving heavy Celery tasks (ffmpeg, TTS, Whisper) to Cloud Run Jobs — finite execution, pay-per-use, env-specific compose, chain dispatch pattern | Video Accessibility | ## Key Architectural Decisions diff --git a/wiki/architecture/cloud-run-jobs-celery.md b/wiki/architecture/cloud-run-jobs-celery.md new file mode 100644 index 0000000..2807217 --- /dev/null +++ b/wiki/architecture/cloud-run-jobs-celery.md @@ -0,0 +1,112 @@ +--- +title: "Cloud Run Jobs for Heavy Celery Tasks" +description: "Moving compute-heavy async tasks (ffmpeg, TTS, Whisper) from Celery workers to Google Cloud Run Jobs" +tags: [architecture, gcp, cloud-run, celery, video-accessibility, workers] +created: 2026-04-29 +updated: 2026-04-29 +--- + +# Cloud Run Jobs for Heavy Celery Tasks + +## Key Takeaways + +- Cloud Run **Jobs** (not Services) are for finite tasks — pay only for execution time, no idle cost +- Heavy workers (`ffmpeg-worker`, `tts-worker`, `whisper-worker`) set to `replicas: 0` on environments that delegate to Cloud Run +- `USE_CELERY_FALLBACK=true` keeps `.delay()` working locally without Cloud Run +- `--redeploy` flag in deploy script = git pull + restart only (no rebuild) — critical for hotfixes +- Task chain (ingest→translate→render) must be dispatched from within the Cloud Run Job process itself — Celery callbacks don't work across the boundary + +## Architecture: Celery → Cloud Run Jobs Migration + +The Video Accessibility project (and similar heavy-AI pipelines) offloads: + +| Task | From | To | +|------|------|----| +| `ingest_and_ai` | Celery `ai-queue` worker | Cloud Run Job | +| `translate_and_synthesize` | Celery `ai-queue` worker | Cloud Run Job | +| `render_accessible_video` | Celery `ai-queue` worker | Cloud Run Job | + +Lightweight tasks (notifications, DB writes) stay on Celery `default` queue — see [[wiki/tech-patterns/redis-celery-worker-queue|redis-celery-worker-queue]]. + +## Environment-Specific Docker Compose + +`docker-compose.optical-dev.yml` exists specifically because optical-dev has 2 CPUs and the prod compose had `cpus: '4.0'`: + +```yaml +# docker-compose.optical-dev.yml +services: + ffmpeg-worker: + deploy: + replicas: 0 # delegated to Cloud Run Jobs + resources: + limits: + cpus: '1.0' # optical-dev only has 2 CPUs + + tts-worker: + deploy: + replicas: 0 + resources: + limits: + cpus: '1.0' +``` + +See [[wiki/concepts/docker-compose-cpu-limits-env|docker-compose-cpu-limits-env]] for the broader pattern. + +## Cloud Run Job Env Vars + +Use `^|^` as delimiter when values contain `=` (JWT secrets, base64 tokens): + +```bash +gcloud run jobs update va-worker \ + --set-env-vars "^|^KEY1=value1|KEY2=jwt=with=equals|KEY3=value3" +``` + +## Local Development Fallback + +```env +USE_CELERY_FALLBACK=true +``` + +When this env var is set, `.delay()` calls route to local Celery workers instead of dispatching Cloud Run Jobs. Keeps local dev working without GCP credentials. + +## Deploy Script Flags + +```bash +./deploy.sh # full rebuild + deploy +./deploy.sh --redeploy # git pull + restart only (no docker build) +``` + +`--redeploy` is critical for hotfixes — build on GCP free-tier takes 10–15 min queued + ~10 min build. Use it whenever the code change doesn't require image rebuild. + +## Cloud Build Image Reference + +``` +europe-west1-docker.pkg.dev/{project}/nexus/va-worker:latest +``` + +## Task Chaining Constraint + +When tasks run as Cloud Run Jobs, Celery callback chains don't work: + +```python +# BROKEN: translate_and_synthesize won't fire after ingest completes +ingest_and_ai.delay(video_id).then(translate_and_synthesize.s()) + +# CORRECT: chain from within the Cloud Run Job process +def run_ingest_job(video_id): + result = ingest_and_ai(video_id) + if result.ok: + dispatch_cloud_run_job("translate_and_synthesize", video_id) +``` + +Alternative: use Cloud Workflows to orchestrate multi-step job chains. + +## Related + +- [[wiki/tech-patterns/redis-celery-worker-queue|redis-celery-worker-queue]] — Celery queue setup and AI queue separation +- [[wiki/concepts/docker-compose-cpu-limits-env|docker-compose-cpu-limits-env]] — why per-environment compose files are needed +- [[wiki/architecture/optical-dev-server-deploy|optical-dev-server-deploy]] — optical-dev server constraints + +## Sources + +- [[daily/2026-04-29.md]] — sessions 21:52 and 20:29 (Video Accessibility Cloud Run migration) diff --git a/wiki/client-knowledge/ford.md b/wiki/client-knowledge/ford.md index d830925..659322f 100644 --- a/wiki/client-knowledge/ford.md +++ b/wiki/client-knowledge/ford.md @@ -7,6 +7,9 @@ created: 2026-04-15 updated: 2026-04-29 --- +> [!info] SSH Alias +> SSH into the Ford QC server using `box-cli` (→ 10.220.176.3). `box-cli-01` does NOT resolve. + # Ford Oliver Agency works with Ford on asset pack automation — QC validation and delivery to Ford's GECHUB SFTP system. @@ -66,22 +69,49 @@ Oliver Agency works with Ford on asset pack automation — QC validation and del > [!warning] Always verify working directory before deploying > The systemd service for Ford QC runs from `ford_qc_git_dev/ford_qc/` — NOT from `FORD_ASSET_PACK_QC` or `FORD_ASSET_PACK_QC_NEW` (legacy directory names that still exist on disk). +**SSH:** `box-cli` (→ 10.220.176.3). Do NOT use `box-cli-01` — it does not resolve. + +**Service names:** +- Dev: `ford-qc-hotfolder.service` +- Production: `ford-qc-hotfolder-PROD.service` + **To find the real working directory before deploying:** ```bash -systemctl show ford-qc --property=ExecStart -# Output: ExecStart={ path=/usr/bin/python3 ; argv[]=/usr/bin/python3 qc_engine.py ; ... WorkingDirectory=/home/oliver/ford_qc_git_dev/ford_qc ; ... } +systemctl show ford-qc-hotfolder-PROD --property=ExecStart +# or +systemctl cat ford-qc-hotfolder-PROD +# WorkingDirectory=/home/box-cli/FORD_SCRIPTS/ford_qc_git_dev/ford_qc ``` -Never assume the working directory from directory names alone. Always use `systemctl show` to confirm. +Never assume the working directory from directory names alone. Always use `systemctl show` or `systemctl cat` to confirm. **Known directories on the Ford QC server (do not confuse):** | Directory | Status | Notes | |-----------|--------|-------| -| `ford_qc_git_dev/ford_qc/` | **Active** | This is where the service runs | -| `FORD_ASSET_PACK_QC/` | Legacy | Old deployment, do not use | -| `FORD_ASSET_PACK_QC_NEW/` | Legacy | Abandoned migration attempt | +| `/home/box-cli/FORD_SCRIPTS/ford_qc_git_dev/ford_qc/` | **Active** | This is where the service runs | +| `/home/box-cli/FORD_SCRIPTS/FORD_ASSET_PACK_QC/` | Legacy | Old deployment, do not use | +| `/home/box-cli/FORD_SCRIPTS/FORD_ASSET_PACK_QC_NEW/` | Stale | Abandoned migration attempt | + +**Git pull with local edits on server:** + +```bash +git stash && git pull && git stash pop +``` + +**ford-gechub-sftp deploy path:** `/home/box-cli/FORD_SCRIPTS/ford-gechub-sftp` (different project) + +## GPAS Zip Naming Convention + +> [!important] Zip filename format changed +> Ford image packs must now end in `_GPAS.zip` — the old `_image.zip` suffix is no longer accepted. + +This is enforced by `zip_filename_check.py`. Packs with incorrect suffixes fail QC immediately at the filename check stage. Update any scripts or templates that generate pack filenames. + +| Old format | New format | +|-----------|-----------| +| `ford_pack_2026_image.zip` | `ford_pack_2026_GPAS.zip` | ## Ford BnP WERS Code Allowlist Pattern diff --git a/wiki/concepts/_index.md b/wiki/concepts/_index.md index ca0c792..a52ff5b 100644 --- a/wiki/concepts/_index.md +++ b/wiki/concepts/_index.md @@ -79,5 +79,10 @@ | [[wiki/concepts/native-track-blob-url]] | Browsers block `data:text/vtt` URIs for `` elements; must use `URL.createObjectURL(new Blob([vttString]))` and revoke in effect destructor | daily/2026-04-29.md | 2026-04-29 | | [[wiki/concepts/claude-code-plugin-marketplace]] | `/plugin marketplace add /` registers GitHub marketplace; then `/plugin install @`; `/plugin add` without marketplace doesn't work for GitHub repos | daily/2026-04-29.md | 2026-04-29 | +| [[wiki/concepts/docker-compose-cpu-limits-env]] | `cpus: '4.0'` fails at `docker compose up` on a 2-CPU server — fix with per-environment compose override files | daily/2026-04-29.md | 2026-04-29 | +| [[wiki/concepts/etag-optimistic-locking]] | ETag/If-Match pattern for concurrent edit protection — backend hash, frontend state (not ref), 412 conflict handling | daily/2026-04-29.md | 2026-04-29 | +| [[wiki/concepts/double-submit-cookie-csrf]] | CSRF for stateless JWT APIs: csrf_token cookie + X-CSRF-Token header; every login path must set both cookies | daily/2026-04-29.md | 2026-04-29 | +| [[wiki/concepts/time-sleep-blocks-asyncio]] | `time.sleep()` inside async FastAPI handlers blocks the entire event loop — replace with `asyncio.sleep()` or `run_in_executor` | daily/2026-04-29.md | 2026-04-29 | + diff --git a/wiki/concepts/docker-compose-cpu-limits-env.md b/wiki/concepts/docker-compose-cpu-limits-env.md new file mode 100644 index 0000000..3a2a84a --- /dev/null +++ b/wiki/concepts/docker-compose-cpu-limits-env.md @@ -0,0 +1,74 @@ +--- +title: "Docker Compose CPU Limits Break Per Environment" +aliases: [docker-cpu-limits, compose-cpu-env, cpus-range-error] +tags: [docker, docker-compose, deployment, devops, gotcha] +sources: + - "daily/2026-04-29.md" +created: 2026-04-29 +updated: 2026-04-29 +--- + +# Docker Compose CPU Limits Break Per Environment + +Setting `cpus: '4.0'` in a prod `docker-compose.yml` and then running `docker compose up` on a server with only 2 CPUs causes a hard error. The fix is a per-environment compose override file that matches the target server's actual CPU count. + +## Key Points + +- Error message: `"range of CPUs is from 0.01 to 2.00"` — appears at `docker compose up`, NOT at `docker compose build` +- The build succeeds silently; the failure only surfaces at container startup +- prod compose is the canonical config; environment-specific files override resource limits only +- Naming convention: `docker-compose.{env}.yml` (e.g. `docker-compose.optical-dev.yml`) + +## Details + +The error occurs because Docker enforces CPU limits against the host's actual CPU count at container start time, not at image build time. A prod file with `cpus: '4.0'` is valid on a 4-core prod server but fails immediately on a 2-core staging server. + +**Prod compose (canonical):** + +```yaml +services: + ffmpeg-worker: + deploy: + resources: + limits: + cpus: '4.0' # valid on 4-core prod + memory: 8G +``` + +**optical-dev override:** + +```yaml +# docker-compose.optical-dev.yml +services: + ffmpeg-worker: + deploy: + replicas: 0 # also disable heavy workers on this env + resources: + limits: + cpus: '1.0' # optical-dev has 2 CPUs — keep 1 for other services + memory: 2G +``` + +**Deploy command using override:** + +```bash +docker compose -f docker-compose.yml -f docker-compose.optical-dev.yml up -d +``` + +## Pattern + +| File | Role | +|------|------| +| `docker-compose.yml` | Canonical config — prod values, all services defined | +| `docker-compose.prod.yml` | Prod-specific overrides (rarely needed if base IS prod) | +| `docker-compose.optical-dev.yml` | optical-dev overrides — reduced CPU/RAM, some services disabled | +| `docker-compose.override.yml` | Local dev — picked up automatically by Docker Compose | + +## Related Concepts + +- [[wiki/architecture/cloud-run-jobs-celery|cloud-run-jobs-celery]] — why optical-dev uses this pattern for heavy workers +- [[wiki/architecture/optical-dev-server-deploy|optical-dev-server-deploy]] — optical-dev server constraints and deployment pattern + +## Sources + +- [[daily/2026-04-29.md]] — session 20:29, `cpus: '4.0'` breaking on 2-CPU optical-dev server diff --git a/wiki/concepts/double-submit-cookie-csrf.md b/wiki/concepts/double-submit-cookie-csrf.md new file mode 100644 index 0000000..798a026 --- /dev/null +++ b/wiki/concepts/double-submit-cookie-csrf.md @@ -0,0 +1,114 @@ +--- +title: "Double Submit Cookie CSRF Pattern for JWT APIs" +aliases: [double-submit-cookie, csrf-jwt, csrf-stateless, x-csrf-token] +tags: [security, csrf, jwt, fastapi, auth, cookies] +sources: + - "daily/2026-04-29.md" +created: 2026-04-29 +updated: 2026-04-29 +--- + +# Double Submit Cookie CSRF Pattern for JWT APIs + +For stateless JWT APIs, the traditional Synchronizer Token Pattern (server-side session stores token) is impossible. The Double Submit Cookie pattern provides CSRF protection without server state. + +## Key Points + +- `csrf_token` cookie is set alongside `refresh_token` at login (same `Set-Cookie` response) +- `/auth/refresh` validates that `X-CSRF-Token` request header matches the `csrf_token` cookie value +- The browser cannot read the cookie value cross-origin (SameSite + HttpOnly still blocks JS reads from other origins) — attacker can't forge the header +- **Critical:** EVERY code path that issues a `refresh_token` MUST also set `csrf_token` — missing even one (e.g. Microsoft OAuth callback) leaves users with refresh token but no CSRF cookie → silent 403s on next token refresh + +## Details + +### Why Not Synchronizer Token? + +Synchronizer Token requires server-side session storage: server generates a token, stores it per-session, validates it on state-changing requests. Stateless JWT APIs have no session → impossible without adding Redis/DB session state. + +### Double Submit Cookie Flow + +``` +POST /auth/login +← Set-Cookie: refresh_token=JWT...; HttpOnly; SameSite=Lax +← Set-Cookie: csrf_token=random_uuid; SameSite=Lax ← NOT HttpOnly (JS reads it) + +POST /auth/refresh +→ Cookie: refresh_token=JWT...; csrf_token=abc123 +→ X-CSRF-Token: abc123 ← JS reads cookie, puts in header +← 200 OK (tokens match) + +# Cross-origin CSRF attack: +POST /auth/refresh (from evil.com) +→ Cookie: refresh_token=JWT... ← browser sends automatically +→ X-CSRF-Token: ??? ← attacker can't read the cookie → cannot forge +← 403 Forbidden +``` + +### FastAPI Implementation + +```python +import secrets + +@router.post("/auth/login") +async def login(response: Response, credentials: LoginRequest): + user = await authenticate(credentials) + refresh_token = create_refresh_token(user.id) + csrf_token = secrets.token_hex(32) + + response.set_cookie("refresh_token", refresh_token, httponly=True, samesite="lax") + response.set_cookie("csrf_token", csrf_token, httponly=False, samesite="lax") + return {"access_token": create_access_token(user.id)} + +@router.post("/auth/refresh") +async def refresh( + request: Request, + x_csrf_token: str | None = Header(None, alias="X-CSRF-Token"), +): + csrf_cookie = request.cookies.get("csrf_token") + if not x_csrf_token or x_csrf_token != csrf_cookie: + raise HTTPException(status_code=403, detail="CSRF validation failed") + # ... issue new access token +``` + +### All Login Paths Must Set CSRF + +```python +# COMMON MISTAKE: Microsoft OAuth callback only sets refresh_token +@router.get("/auth/callback/microsoft") +async def ms_callback(code: str, response: Response): + user = await exchange_ms_code(code) + refresh_token = create_refresh_token(user.id) + response.set_cookie("refresh_token", refresh_token, httponly=True) + # ← MISSING: csrf_token not set here! + # Result: user logs in via MS → no csrf cookie → 403 on first /auth/refresh +``` + +The fix: extract a `set_auth_cookies(response, user_id)` helper that always sets both cookies, call it from every login path. + +### Frontend + +```typescript +// Read csrf_token from cookie (it's NOT httpOnly) +function getCsrfToken(): string | null { + return document.cookie + .split("; ") + .find(row => row.startsWith("csrf_token=")) + ?.split("=")[1] ?? null; +} + +// Include in every refresh request +await fetch("/auth/refresh", { + method: "POST", + headers: { "X-CSRF-Token": getCsrfToken() ?? "" }, + credentials: "include", +}); +``` + +## Related Concepts + +- [[wiki/concepts/etag-optimistic-locking|etag-optimistic-locking]] — another HTTP header-based protection pattern +- [[wiki/tech-patterns/azure-ad-msal-auth|azure-ad-msal-auth]] — Oliver standard auth (MSAL handles CSRF differently via PKCE) + +## Sources + +- [[daily/2026-04-29.md]] — session 19:06, CSRF protection for stateless JWT refresh endpoint diff --git a/wiki/concepts/etag-optimistic-locking.md b/wiki/concepts/etag-optimistic-locking.md new file mode 100644 index 0000000..99897b8 --- /dev/null +++ b/wiki/concepts/etag-optimistic-locking.md @@ -0,0 +1,102 @@ +--- +title: "ETag / If-Match Optimistic Locking" +aliases: [etag-locking, optimistic-locking, if-match-patch, 412-precondition] +tags: [fastapi, react, http, concurrency, optimistic-locking, api-design] +sources: + - "daily/2026-04-29.md" +created: 2026-04-29 +updated: 2026-04-29 +--- + +# ETag / If-Match Optimistic Locking + +Prevents two users from silently overwriting each other's edits without server-side locking. The backend generates an ETag from a content hash; the frontend sends it back on write; stale writes are rejected with 412. + +## Key Points + +- Backend generates `ETag` from content hash (e.g. `md5(content)` or document version) +- Frontend stores ETag in **React state** (not a ref) — so React Query invalidation triggers a re-fetch on conflict +- PATCH request includes `If-Match: {etag}` header +- Backend checks: if ETag doesn't match current document version → `412 Precondition Failed` +- Frontend `412` handler shows user-visible conflict message and re-fetches latest content + +## Details + +### Backend (FastAPI) + +```python +from hashlib import md5 +from fastapi import Header, HTTPException + +@router.get("/vtt/{segment_id}") +async def get_vtt(segment_id: str): + doc = await db.vtt_segments.find_one({"_id": segment_id}) + content_hash = md5(doc["content"].encode()).hexdigest() + return JSONResponse( + content=doc, + headers={"ETag": f'"{content_hash}"'} + ) + +@router.patch("/vtt/{segment_id}") +async def update_vtt( + segment_id: str, + payload: VttUpdate, + if_match: str | None = Header(None, alias="If-Match"), +): + doc = await db.vtt_segments.find_one({"_id": segment_id}) + current_hash = f'"{md5(doc["content"].encode()).hexdigest()}"' + if if_match and if_match != current_hash: + raise HTTPException(status_code=412, detail="Content modified by another user") + # ... proceed with update +``` + +### Frontend (React + React Query) + +```typescript +// Store ETag in state, not ref — ensures re-render on update +const [etag, setEtag] = useState(null); + +// On fetch, capture the ETag +const { data } = useQuery(["vtt", segmentId], async () => { + const res = await fetch(`/api/vtt/${segmentId}`); + setEtag(res.headers.get("ETag")); + return res.json(); +}); + +// On save, send If-Match header +const handleSave = async (content: string) => { + const res = await fetch(`/api/vtt/${segmentId}`, { + method: "PATCH", + headers: { + "Content-Type": "application/json", + ...(etag ? { "If-Match": etag } : {}), + }, + body: JSON.stringify({ content }), + }); + + if (res.status === 412) { + alert("Someone else edited this segment. Refreshing to show latest version."); + queryClient.invalidateQueries(["vtt", segmentId]); + return; + } +}; +``` + +### Why State, Not Ref + +If ETag is stored in a `useRef`, React Query invalidation won't trigger a re-render → the stale ETag persists → the next save attempt uses the old ETag again. Storing in `useState` ensures the component re-renders after invalidation and the fresh ETag is picked up. + +## Use Cases + +- Collaborative editors (VTT subtitles, document editors) +- Any resource where two users might edit simultaneously (admin panels, CMS) +- Long-lived forms where network latency could cause version skew + +## Related Concepts + +- [[wiki/concepts/double-submit-cookie-csrf|double-submit-cookie-csrf]] — another HTTP security header pattern for stateless APIs +- [[wiki/tech-patterns/fastapi-python-docker|fastapi-python-docker]] — FastAPI patterns + +## Sources + +- [[daily/2026-04-29.md]] — session 19:06, VTT editor concurrent edit protection diff --git a/wiki/concepts/time-sleep-blocks-asyncio.md b/wiki/concepts/time-sleep-blocks-asyncio.md new file mode 100644 index 0000000..8636246 --- /dev/null +++ b/wiki/concepts/time-sleep-blocks-asyncio.md @@ -0,0 +1,97 @@ +--- +title: "time.sleep() Blocks the Entire asyncio Event Loop" +aliases: [time-sleep-async, asyncio-blocking, sync-sleep-fastapi, asyncio-sleep] +tags: [python, fastapi, asyncio, concurrency, gotcha, performance] +sources: + - "daily/2026-04-29.md" +created: 2026-04-29 +updated: 2026-04-29 +--- + +# time.sleep() Blocks the Entire asyncio Event Loop + +`time.sleep(N)` called inside any `async def` handler freezes the entire asyncio event loop for the duration — no other requests can be processed. It appears to work in local tests (single request) but silently kills concurrency in production. + +## Key Points + +- `time.sleep()` is a **synchronous blocking call** — it blocks the OS thread, which blocks the asyncio event loop +- `asyncio.sleep()` is the correct replacement — it yields control back to the event loop while waiting +- For CPU-bound or I/O-bound sync code, use `asyncio.run_in_executor()` to run in a threadpool +- Easy to miss because single-request tests always pass — the bug only appears under concurrent load + +## Details + +### The Problem + +```python +# WRONG — blocks all concurrent requests for 5 seconds +@router.get("/process") +async def process_something(): + await do_first_step() + time.sleep(5) # ← entire event loop frozen here + await do_second_step() + return {"status": "done"} +``` + +While `time.sleep(5)` runs, every other incoming request waits. On a server handling 50 concurrent users, this compounds into multi-second stalls for every user. + +### Correct Replacements + +**1. Async wait (most common case):** + +```python +import asyncio + +@router.get("/process") +async def process_something(): + await do_first_step() + await asyncio.sleep(5) # ← yields to event loop, other requests continue + await do_second_step() + return {"status": "done"} +``` + +**2. Sync blocking work in threadpool:** + +```python +import asyncio +from concurrent.futures import ThreadPoolExecutor + +def heavy_sync_work(): + time.sleep(5) # or any blocking I/O + return result + +@router.get("/process") +async def process_something(): + loop = asyncio.get_event_loop() + result = await loop.run_in_executor(None, heavy_sync_work) + return {"result": result} +``` + +**3. FastAPI's `def` (not `async def`) routes — automatic threadpool:** + +```python +# FastAPI automatically runs non-async routes in a threadpool +@router.get("/process") +def process_something(): # ← no async keyword + time.sleep(5) # safe here — runs in threadpool + return {"status": "done"} +``` + +### Code Review Checklist + +Search for `time.sleep` as part of any FastAPI code review: + +```bash +grep -rn "time\.sleep" app/routers/ app/services/ +``` + +Any hit inside an `async def` function is a bug. + +## Related Concepts + +- [[wiki/concepts/asyncio-contextvar-task-boundary|asyncio-contextvar-task-boundary]] — other asyncio gotchas in FastAPI +- [[wiki/tech-patterns/fastapi-python-docker|fastapi-python-docker]] — FastAPI best practices + +## Sources + +- [[daily/2026-04-29.md]] — session 17:51, blocking sleep found in async FastAPI handler diff --git a/wiki/log.md b/wiki/log.md index 00a6805..8780e05 100644 --- a/wiki/log.md +++ b/wiki/log.md @@ -141,3 +141,9 @@ - Articles created: [[wiki/concepts/bash-and-or-short-circuit]], [[wiki/concepts/python-iso-z-suffix]], [[wiki/concepts/gemini-conversation-cost-scaling]] - Articles updated: [[wiki/concepts/_index]] (concepts 51→54); [[wiki/_master-index]] (concepts 51→54) - Note: Kling tech-pattern article already up-to-date from prior session; dotfiles Ghostty gotchas deferred (low reuse value vs 3 broadly applicable concepts) + +## [2026-04-29T23:00:00+01:00] compile | 2026-04-29.md (pass 2) +- Source: daily/2026-04-29.md +- Articles created: [[wiki/architecture/cloud-run-jobs-celery]], [[wiki/concepts/docker-compose-cpu-limits-env]], [[wiki/concepts/etag-optimistic-locking]], [[wiki/concepts/double-submit-cookie-csrf]], [[wiki/concepts/time-sleep-blocks-asyncio]] +- Articles updated: [[wiki/client-knowledge/ford]] (SSH alias box-cli/not box-cli-01, full directory paths with /home/box-cli/FORD_SCRIPTS/ prefix, service names dev vs prod, GPAS zip naming, git stash deploy pattern, ford-gechub-sftp path) +- Index updates: [[wiki/concepts/_index]] (69→73); [[wiki/architecture/_index]] (10→11); [[wiki/_master-index]] (concepts 69→73, architecture 10→11)