From 27ba48d181d4e89f70073fb0061311b7f38053f9 Mon Sep 17 00:00:00 2001 From: Anthony GAEREMYNCK <1@anthony.sh> Date: Tue, 11 Aug 2026 14:13:38 +0200 Subject: [PATCH] feat: gate download options on real-time yt-dlp video analysis Options (quality, subtitles) are now derived from a yt-dlp -J probe of the submitted URL instead of a static list, so users can't pick combinations the source video doesn't actually support. Submission is blocked until the probe succeeds. - New /api/probe endpoint + src/lib/ytdlp-probe.ts, with its own rate limiter (rate-limit.ts refactored into a createRateLimiter factory). - New options: subtitle language selection, clip start/end trim, MP3 audio quality. - Fixed buildYtdlpArgs: format=mp3 never triggered audio extraction (-x/--audio-format), and quality=best never applied --merge-output-format, so the chosen container had no real effect. - Cross-platform yt-dlp invocation: the bundled bin/yt-dlp is a Python zipapp relying on a shebang, which Windows' spawn() can't run directly. buildYtdlpCommand() runs it through `python` on Windows and execs it directly on Linux/o2switch, where the shebang works natively. Co-Authored-By: Claude Sonnet 5 --- config/app.config.ts | 3 + messages/en.json | 14 +- messages/es.json | 14 +- messages/fr.json | 14 +- messages/it.json | 14 +- .../migration.sql | 5 + prisma/schema.prisma | 4 + src/app/api/downloads/__tests__/post.test.ts | 40 ++- src/app/api/downloads/route.ts | 26 +- src/app/api/probe/__tests__/post.test.ts | 63 ++++ src/app/api/probe/route.ts | 31 ++ src/components/SubmitForm.tsx | 285 +++++++++++++++--- src/lib/__tests__/rate-limit.test.ts | 36 ++- src/lib/__tests__/ytdlp-command.test.ts | 50 +++ src/lib/__tests__/ytdlp-probe.test.ts | 76 +++++ src/lib/__tests__/ytdlp.test.ts | 84 +++++- src/lib/rate-limit.ts | 39 ++- src/lib/request-ip.ts | 5 + src/lib/ytdlp-options.ts | 3 + src/lib/ytdlp-probe.ts | 125 ++++++++ src/lib/ytdlp.ts | 57 +++- worker/__tests__/processor.test.ts | 4 + worker/processor.ts | 16 +- 23 files changed, 897 insertions(+), 111 deletions(-) create mode 100644 prisma/migrations/20260811115557_add_advanced_download_options/migration.sql create mode 100644 src/app/api/probe/__tests__/post.test.ts create mode 100644 src/app/api/probe/route.ts create mode 100644 src/lib/__tests__/ytdlp-command.test.ts create mode 100644 src/lib/__tests__/ytdlp-probe.test.ts create mode 100644 src/lib/request-ip.ts create mode 100644 src/lib/ytdlp-options.ts create mode 100644 src/lib/ytdlp-probe.ts diff --git a/config/app.config.ts b/config/app.config.ts index e7d665c..a83e126 100644 --- a/config/app.config.ts +++ b/config/app.config.ts @@ -7,5 +7,8 @@ export const config = { STORAGE_PATH: process.env.STORAGE_PATH ?? '', RATE_LIMIT_MAX: 5, RATE_LIMIT_WINDOW_MS: 3_600_000, + PROBE_RATE_LIMIT_MAX: 20, + PROBE_RATE_LIMIT_WINDOW_MS: 3_600_000, + PROBE_TIMEOUT_MS: 20_000, BIN_DIR: process.env.BIN_DIR ?? path.join(process.cwd(), 'bin'), } as const diff --git a/messages/en.json b/messages/en.json index dd2cb74..e4f5498 100644 --- a/messages/en.json +++ b/messages/en.json @@ -9,13 +9,23 @@ "urlLabel": "Video URL", "format": "Format", "quality": "Quality", - "subtitles": "Subtitles (fr, en)", + "subtitles": "Subtitles", + "subtitleLanguages": "Subtitle languages", "advanced": "Advanced options", "formats": "mp4 · mp3 · webm · mkv", "submit": "Download", "submitting": "Sending...", "errorRateLimit": "Too many submissions. Try again in an hour.", - "errorGeneric": "Submission error." + "errorGeneric": "Submission error.", + "analyzing": "Analyzing video...", + "probeError": "Couldn't analyze this video. Check the link or try again.", + "retry": "Retry", + "audioOnlySource": "Audio-only source", + "clipRange": "Clip (optional)", + "clipStart": "Start (s)", + "clipEnd": "End (s)", + "audioQuality": "Audio quality", + "videoLength": "Length: {duration}" }, "status": { "heading": "Download status", diff --git a/messages/es.json b/messages/es.json index c97e60f..ad12fce 100644 --- a/messages/es.json +++ b/messages/es.json @@ -9,13 +9,23 @@ "urlLabel": "URL del vídeo", "format": "Formato", "quality": "Calidad", - "subtitles": "Subtítulos (fr, en)", + "subtitles": "Subtítulos", + "subtitleLanguages": "Idiomas de los subtítulos", "advanced": "Opciones avanzadas", "formats": "mp4 · mp3 · webm · mkv", "submit": "Descargar", "submitting": "Enviando...", "errorRateLimit": "Demasiadas peticiones. Inténtalo de nuevo en una hora.", - "errorGeneric": "Error al enviar." + "errorGeneric": "Error al enviar.", + "analyzing": "Analizando el vídeo...", + "probeError": "No se pudo analizar este vídeo. Comprueba el enlace o inténtalo de nuevo.", + "retry": "Reintentar", + "audioOnlySource": "Fuente solo de audio", + "clipRange": "Fragmento (opcional)", + "clipStart": "Inicio (s)", + "clipEnd": "Fin (s)", + "audioQuality": "Calidad de audio", + "videoLength": "Duración: {duration}" }, "status": { "heading": "Estado de la descarga", diff --git a/messages/fr.json b/messages/fr.json index 2d50890..2e45a72 100644 --- a/messages/fr.json +++ b/messages/fr.json @@ -9,13 +9,23 @@ "urlLabel": "URL de la vidéo", "format": "Format", "quality": "Qualité", - "subtitles": "Sous-titres (fr, en)", + "subtitles": "Sous-titres", + "subtitleLanguages": "Langues des sous-titres", "advanced": "Options avancées", "formats": "mp4 · mp3 · webm · mkv", "submit": "Télécharger", "submitting": "Envoi en cours...", "errorRateLimit": "Trop de soumissions. Réessayez dans une heure.", - "errorGeneric": "Erreur lors de la soumission." + "errorGeneric": "Erreur lors de la soumission.", + "analyzing": "Analyse de la vidéo...", + "probeError": "Impossible d'analyser cette vidéo. Vérifiez le lien ou réessayez.", + "retry": "Réessayer", + "audioOnlySource": "Source audio uniquement", + "clipRange": "Extrait (optionnel)", + "clipStart": "Début (s)", + "clipEnd": "Fin (s)", + "audioQuality": "Qualité audio", + "videoLength": "Durée : {duration}" }, "status": { "heading": "Statut du téléchargement", diff --git a/messages/it.json b/messages/it.json index 5c05008..dfdd38e 100644 --- a/messages/it.json +++ b/messages/it.json @@ -9,13 +9,23 @@ "urlLabel": "URL del video", "format": "Formato", "quality": "Qualità", - "subtitles": "Sottotitoli (fr, en)", + "subtitles": "Sottotitoli", + "subtitleLanguages": "Lingue dei sottotitoli", "advanced": "Opzioni avanzate", "formats": "mp4 · mp3 · webm · mkv", "submit": "Scarica", "submitting": "Invio in corso...", "errorRateLimit": "Troppe richieste. Riprova tra un'ora.", - "errorGeneric": "Errore durante l'invio." + "errorGeneric": "Errore durante l'invio.", + "analyzing": "Analisi del video...", + "probeError": "Impossibile analizzare questo video. Controlla il link o riprova.", + "retry": "Riprova", + "audioOnlySource": "Sorgente solo audio", + "clipRange": "Estratto (opzionale)", + "clipStart": "Inizio (s)", + "clipEnd": "Fine (s)", + "audioQuality": "Qualità audio", + "videoLength": "Durata: {duration}" }, "status": { "heading": "Stato del download", diff --git a/prisma/migrations/20260811115557_add_advanced_download_options/migration.sql b/prisma/migrations/20260811115557_add_advanced_download_options/migration.sql new file mode 100644 index 0000000..1a7be84 --- /dev/null +++ b/prisma/migrations/20260811115557_add_advanced_download_options/migration.sql @@ -0,0 +1,5 @@ +-- AlterTable +ALTER TABLE `Download` ADD COLUMN `audioQuality` VARCHAR(191) NULL, + ADD COLUMN `clipEnd` INTEGER NULL, + ADD COLUMN `clipStart` INTEGER NULL, + ADD COLUMN `subtitleLangs` TEXT NULL; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 3ba13f6..993eafd 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -14,6 +14,10 @@ model Download { format String quality String subtitles Boolean @default(false) + subtitleLangs String? @db.Text + clipStart Int? + clipEnd Int? + audioQuality String? extraArgs String? @db.Text filePath String? @db.Text diff --git a/src/app/api/downloads/__tests__/post.test.ts b/src/app/api/downloads/__tests__/post.test.ts index 3035822..b9cd211 100644 --- a/src/app/api/downloads/__tests__/post.test.ts +++ b/src/app/api/downloads/__tests__/post.test.ts @@ -1,14 +1,16 @@ import { POST } from '../route' import { prisma } from '@/lib/prisma' -import * as rateLimit from '@/lib/rate-limit' +import { downloadRateLimiter } from '@/lib/rate-limit' jest.mock('@/lib/prisma', () => ({ prisma: { download: { create: jest.fn() } }, })) -jest.mock('@/lib/rate-limit') +jest.mock('@/lib/rate-limit', () => ({ + downloadRateLimiter: { isLimited: jest.fn() }, +})) const mockCreate = prisma.download.create as jest.Mock -const mockIsRateLimited = rateLimit.isRateLimited as jest.Mock +const mockIsLimited = downloadRateLimiter.isLimited as jest.Mock function req(body: object, ip = '1.2.3.4') { return new Request('http://localhost/api/downloads', { @@ -20,12 +22,12 @@ function req(body: object, ip = '1.2.3.4') { describe('POST /api/downloads', () => { beforeEach(() => { - mockIsRateLimited.mockReturnValue(false) + mockIsLimited.mockReturnValue(false) mockCreate.mockResolvedValue({ uuid: 'test-uuid' }) }) it('returns 429 when rate limited', async () => { - mockIsRateLimited.mockReturnValue(true) + mockIsLimited.mockReturnValue(true) const res = await POST(req({ url: 'https://y.com', format: 'mp4', quality: 'best', subtitles: false })) expect(res.status).toBe(429) }) @@ -40,6 +42,14 @@ describe('POST /api/downloads', () => { expect(res.status).toBe(400) }) + it('returns 400 when clipEnd is not greater than clipStart', async () => { + const res = await POST(req({ + url: 'https://y.com', format: 'mp4', quality: 'best', subtitles: false, + clipStart: 30, clipEnd: 10, + })) + expect(res.status).toBe(400) + }) + it('returns 201 with uuid on success', async () => { const res = await POST(req({ url: 'https://youtube.com/watch?v=abc', format: 'mp4', quality: 'best', subtitles: false })) expect(res.status).toBe(201) @@ -58,4 +68,24 @@ describe('POST /api/downloads', () => { }), }) }) + + it('stores subtitleLangs as a comma-joined string', async () => { + await POST(req({ + url: 'https://y.com', format: 'mp4', quality: 'best', subtitles: true, + subtitleLangs: ['fr', 'en'], + })) + expect(mockCreate).toHaveBeenCalledWith({ + data: expect.objectContaining({ subtitleLangs: 'fr,en' }), + }) + }) + + it('stores clip range and audio quality when provided', async () => { + await POST(req({ + url: 'https://y.com', format: 'mp3', quality: 'best', subtitles: false, + clipStart: 10, clipEnd: 30, audioQuality: '192', + })) + expect(mockCreate).toHaveBeenCalledWith({ + data: expect.objectContaining({ clipStart: 10, clipEnd: 30, audioQuality: '192' }), + }) + }) }) diff --git a/src/app/api/downloads/route.ts b/src/app/api/downloads/route.ts index 5088dcb..d5ebea6 100644 --- a/src/app/api/downloads/route.ts +++ b/src/app/api/downloads/route.ts @@ -1,12 +1,12 @@ import { NextRequest, NextResponse } from 'next/server' import { prisma } from '@/lib/prisma' -import { isRateLimited } from '@/lib/rate-limit' +import { downloadRateLimiter } from '@/lib/rate-limit' +import { getClientIp } from '@/lib/request-ip' export async function POST(req: NextRequest) { - const ip = - req.headers.get('x-forwarded-for')?.split(',')[0].trim() ?? '0.0.0.0' + const ip = getClientIp(req) - if (isRateLimited(ip)) { + if (downloadRateLimiter.isLimited(ip)) { return NextResponse.json({ error: 'Rate limit exceeded' }, { status: 429 }) } @@ -24,12 +24,30 @@ export async function POST(req: NextRequest) { ) } + const clipStart = body.clipStart != null ? Number(body.clipStart) : null + const clipEnd = body.clipEnd != null ? Number(body.clipEnd) : null + + if (clipStart != null && clipEnd != null && clipEnd <= clipStart) { + return NextResponse.json( + { error: 'Invalid clip range: clipEnd must be greater than clipStart' }, + { status: 400 } + ) + } + + const subtitleLangs: string[] | null = Array.isArray(body.subtitleLangs) && body.subtitleLangs.length > 0 + ? body.subtitleLangs.map(String) + : null + const download = await prisma.download.create({ data: { url: String(body.url), format: String(body.format), quality: String(body.quality), subtitles: Boolean(body.subtitles), + subtitleLangs: subtitleLangs ? subtitleLangs.join(',') : null, + clipStart, + clipEnd, + audioQuality: body.audioQuality ? String(body.audioQuality) : null, extraArgs: body.extraArgs ?? null, ipAddress: ip, }, diff --git a/src/app/api/probe/__tests__/post.test.ts b/src/app/api/probe/__tests__/post.test.ts new file mode 100644 index 0000000..8d89532 --- /dev/null +++ b/src/app/api/probe/__tests__/post.test.ts @@ -0,0 +1,63 @@ +import { POST } from '../route' +import { probeRateLimiter } from '@/lib/rate-limit' +import { runYtdlpProbe } from '@/lib/ytdlp-probe' + +jest.mock('@/lib/rate-limit', () => ({ + probeRateLimiter: { isLimited: jest.fn() }, +})) +jest.mock('@/lib/ytdlp-probe', () => ({ + runYtdlpProbe: jest.fn(), +})) + +const mockIsLimited = probeRateLimiter.isLimited as jest.Mock +const mockRunProbe = runYtdlpProbe as jest.Mock + +function req(body: object, ip = '1.2.3.4') { + return new Request('http://localhost/api/probe', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-forwarded-for': ip }, + body: JSON.stringify(body), + }) +} + +describe('POST /api/probe', () => { + beforeEach(() => { + mockIsLimited.mockReturnValue(false) + }) + + it('returns 429 when rate limited', async () => { + mockIsLimited.mockReturnValue(true) + const res = await POST(req({ url: 'https://youtube.com/watch?v=abc' })) + expect(res.status).toBe(429) + }) + + it('returns 400 when url is missing', async () => { + const res = await POST(req({})) + expect(res.status).toBe(400) + }) + + it('returns 400 when url is not a valid URL', async () => { + const res = await POST(req({ url: 'not-a-url' })) + expect(res.status).toBe(400) + }) + + it('returns 200 with capabilities on success', async () => { + const capabilities = { + title: 'Some video', + durationSeconds: 100, + isAudioOnly: false, + availableQualities: ['best', '720p'], + subtitleLangs: ['en'], + } + mockRunProbe.mockResolvedValue(capabilities) + const res = await POST(req({ url: 'https://youtube.com/watch?v=abc' })) + expect(res.status).toBe(200) + expect(await res.json()).toEqual(capabilities) + }) + + it('returns 422 when the probe fails', async () => { + mockRunProbe.mockRejectedValue(new Error('boom')) + const res = await POST(req({ url: 'https://youtube.com/watch?v=abc' })) + expect(res.status).toBe(422) + }) +}) diff --git a/src/app/api/probe/route.ts b/src/app/api/probe/route.ts new file mode 100644 index 0000000..075a92c --- /dev/null +++ b/src/app/api/probe/route.ts @@ -0,0 +1,31 @@ +import { NextRequest, NextResponse } from 'next/server' +import { probeRateLimiter } from '@/lib/rate-limit' +import { getClientIp } from '@/lib/request-ip' +import { runYtdlpProbe } from '@/lib/ytdlp-probe' + +export async function POST(req: NextRequest) { + const ip = getClientIp(req) + + if (probeRateLimiter.isLimited(ip)) { + return NextResponse.json({ error: 'Rate limit exceeded' }, { status: 429 }) + } + + const body = await req.json().catch(() => null) + + if (!body?.url || typeof body.url !== 'string') { + return NextResponse.json({ error: 'Missing required field: url' }, { status: 400 }) + } + + try { + new URL(body.url) + } catch { + return NextResponse.json({ error: 'Invalid URL' }, { status: 400 }) + } + + try { + const capabilities = await runYtdlpProbe(body.url) + return NextResponse.json(capabilities) + } catch { + return NextResponse.json({ error: 'PROBE_FAILED' }, { status: 422 }) + } +} diff --git a/src/components/SubmitForm.tsx b/src/components/SubmitForm.tsx index 844557b..d5cd796 100644 --- a/src/components/SubmitForm.tsx +++ b/src/components/SubmitForm.tsx @@ -1,15 +1,28 @@ 'use client' -import { useState, FormEvent } from 'react' -import { useTranslations } from 'next-intl' +import { useEffect, useRef, useState, FormEvent } from 'react' +import { useTranslations, useLocale } from 'next-intl' import { useRouter } from '@/navigation' +import { FORMATS, AUDIO_QUALITIES } from '@/lib/ytdlp-options' -const FORMATS = ['mp4', 'mp3', 'webm', 'mkv'] -const QUALITIES = ['best', '1080p', '720p', '480p', '360p'] +type ProbeCapabilities = { + title: string + durationSeconds: number | null + isAudioOnly: boolean + availableQualities: string[] + subtitleLangs: string[] +} -function Spinner() { +type ProbeState = + | { status: 'idle' } + | { status: 'debouncing' } + | { status: 'probing' } + | { status: 'ready'; data: ProbeCapabilities } + | { status: 'error' } + +function Spinner({ className = 'w-4 h-4' }: { className?: string }) { return ( - + @@ -19,19 +32,100 @@ function Spinner() { const selectClass = 'rounded-lg border border-gray-200 dark:border-slate-700 bg-white dark:bg-slate-900 px-2 py-1 text-gray-900 dark:text-gray-50 text-sm outline-none focus:ring-2 focus:ring-violet-500' +const numberInputClass = + 'w-24 rounded-lg border border-gray-200 dark:border-slate-700 bg-white dark:bg-slate-900 px-2 py-1 text-gray-900 dark:text-gray-50 text-sm outline-none focus:ring-2 focus:ring-violet-500' + +function isValidUrl(value: string): boolean { + try { + new URL(value) + return true + } catch { + return false + } +} + +function langLabel(code: string, locale: string): string { + try { + return new Intl.DisplayNames([locale], { type: 'language' }).of(code) ?? code + } catch { + return code + } +} + +function formatDuration(seconds: number): string { + const m = Math.floor(seconds / 60) + const s = Math.floor(seconds % 60) + return `${m}:${s.toString().padStart(2, '0')}` +} + export function SubmitForm({ examplePlaceholder }: { examplePlaceholder?: string } = {}) { const t = useTranslations('home') + const locale = useLocale() const router = useRouter() + const [url, setUrl] = useState('') + const [probe, setProbe] = useState({ status: 'idle' }) + const [retryToken, setRetryToken] = useState(0) + const [format, setFormat] = useState('mp4') const [quality, setQuality] = useState('best') const [subtitles, setSubtitles] = useState(false) + const [subtitleLangs, setSubtitleLangs] = useState([]) + const [audioQuality, setAudioQuality] = useState('best') + const [clipStart, setClipStart] = useState('') + const [clipEnd, setClipEnd] = useState('') const [extraArgs, setExtraArgs] = useState('') + const [error, setError] = useState(null) const [loading, setLoading] = useState(false) + const abortRef = useRef(null) + + useEffect(() => { + abortRef.current?.abort() + + if (!isValidUrl(url)) { + setProbe({ status: 'idle' }) + return + } + + setProbe({ status: 'debouncing' }) + + const debounce = setTimeout(() => { + const controller = new AbortController() + abortRef.current = controller + setProbe({ status: 'probing' }) + + fetch('/api/probe', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ url }), + signal: controller.signal, + }) + .then(async (res) => { + if (!res.ok) throw new Error('probe failed') + const data = (await res.json()) as ProbeCapabilities + setProbe({ status: 'ready', data }) + + if (data.isAudioOnly) { + setFormat('mp3') + } + setQuality((prev) => (data.availableQualities.includes(prev) ? prev : 'best')) + setSubtitleLangs(data.subtitleLangs) + }) + .catch((err) => { + if (err.name === 'AbortError') return + setProbe({ status: 'error' }) + }) + }, 500) + + return () => clearTimeout(debounce) + }, [url, retryToken]) + async function handleSubmit(e: FormEvent) { e.preventDefault() + if (probe.status !== 'ready') return + setError(null) setLoading(true) @@ -43,6 +137,10 @@ export function SubmitForm({ examplePlaceholder }: { examplePlaceholder?: string format, quality, subtitles, + subtitleLangs: subtitles && subtitleLangs.length ? subtitleLangs : null, + audioQuality: format === 'mp3' ? audioQuality : null, + clipStart: clipStart !== '' ? Number(clipStart) : null, + clipEnd: clipEnd !== '' ? Number(clipEnd) : null, extraArgs: extraArgs.trim() || null, }), }) @@ -63,6 +161,9 @@ export function SubmitForm({ examplePlaceholder }: { examplePlaceholder?: string router.push(`/status/${uuid}`) } + const capabilities = probe.status === 'ready' ? probe.data : null + const submitDisabled = loading || probe.status !== 'ready' + return (
{/* URL input with inline submit button */} @@ -80,7 +181,7 @@ export function SubmitForm({ examplePlaceholder }: { examplePlaceholder?: string />
- + {t('retry')} + +
+ )} - + {capabilities && ( +
+ {capabilities.isAudioOnly ? ( + + {t('audioOnlySource')} + + ) : ( + <> + - + + + )} -
- - {t('advanced')} - - setExtraArgs(e.target.value)} - placeholder='["--sponsorblock-remove","all"]' - className="mt-2 w-full rounded-lg border border-gray-200 dark:border-slate-700 bg-white dark:bg-slate-900 px-3 py-2 text-sm text-gray-900 dark:text-gray-50 outline-none focus:ring-2 focus:ring-violet-500" - /> -
-
+ {format === 'mp3' && ( + + )} + + {capabilities.subtitleLangs.length > 0 && ( + + )} + + {subtitles && capabilities.subtitleLangs.length > 0 && ( +
+ {t('subtitleLanguages')}: + {capabilities.subtitleLangs.map((lang) => ( + + ))} +
+ )} + +
+ {t('clipRange')} + + + {capabilities.durationSeconds != null && ( + + {t('videoLength', { duration: formatDuration(capabilities.durationSeconds) })} + + )} +
+ +
+ + {t('advanced')} + + setExtraArgs(e.target.value)} + placeholder='["--sponsorblock-remove","all"]' + className="mt-2 w-full rounded-lg border border-gray-200 dark:border-slate-700 bg-white dark:bg-slate-900 px-3 py-2 text-sm text-gray-900 dark:text-gray-50 outline-none focus:ring-2 focus:ring-violet-500" + /> +
+ + )} {error && (

{error}

diff --git a/src/lib/__tests__/rate-limit.test.ts b/src/lib/__tests__/rate-limit.test.ts index 8089c2e..7eca0a4 100644 --- a/src/lib/__tests__/rate-limit.test.ts +++ b/src/lib/__tests__/rate-limit.test.ts @@ -1,41 +1,53 @@ -import { isRateLimited } from '../rate-limit' +import { createRateLimiter } from '../rate-limit' beforeEach(() => { jest.useFakeTimers() - jest.resetModules() }) afterEach(() => { jest.useRealTimers() }) -describe('isRateLimited', () => { +describe('createRateLimiter', () => { it('allows requests under the limit', () => { + const limiter = createRateLimiter(5, 3_600_000) const ip = '1.2.3.4' for (let i = 0; i < 5; i++) { - expect(isRateLimited(ip)).toBe(false) + expect(limiter.isLimited(ip)).toBe(false) } }) it('blocks the 6th request within the window', () => { + const limiter = createRateLimiter(5, 3_600_000) const ip = '10.0.0.1' - for (let i = 0; i < 5; i++) isRateLimited(ip) - expect(isRateLimited(ip)).toBe(true) + for (let i = 0; i < 5; i++) limiter.isLimited(ip) + expect(limiter.isLimited(ip)).toBe(true) }) it('resets after the window expires', () => { + const limiter = createRateLimiter(5, 3_600_000) const ip = '10.0.0.2' - for (let i = 0; i < 5; i++) isRateLimited(ip) - expect(isRateLimited(ip)).toBe(true) + for (let i = 0; i < 5; i++) limiter.isLimited(ip) + expect(limiter.isLimited(ip)).toBe(true) jest.advanceTimersByTime(3_600_001) - expect(isRateLimited(ip)).toBe(false) + expect(limiter.isLimited(ip)).toBe(false) }) it('tracks different IPs independently', () => { + const limiter = createRateLimiter(5, 3_600_000) const ipA = '192.168.1.1' const ipB = '192.168.1.2' - for (let i = 0; i < 5; i++) isRateLimited(ipA) - expect(isRateLimited(ipA)).toBe(true) - expect(isRateLimited(ipB)).toBe(false) + for (let i = 0; i < 5; i++) limiter.isLimited(ipA) + expect(limiter.isLimited(ipA)).toBe(true) + expect(limiter.isLimited(ipB)).toBe(false) + }) + + it('tracks separate limiter instances independently', () => { + const limiterA = createRateLimiter(1, 3_600_000) + const limiterB = createRateLimiter(1, 3_600_000) + const ip = '5.5.5.5' + expect(limiterA.isLimited(ip)).toBe(false) + expect(limiterA.isLimited(ip)).toBe(true) + expect(limiterB.isLimited(ip)).toBe(false) }) }) diff --git a/src/lib/__tests__/ytdlp-command.test.ts b/src/lib/__tests__/ytdlp-command.test.ts new file mode 100644 index 0000000..fb35c69 --- /dev/null +++ b/src/lib/__tests__/ytdlp-command.test.ts @@ -0,0 +1,50 @@ +const mockExistsSync = jest.fn() +jest.mock('fs', () => ({ existsSync: (...args: unknown[]) => mockExistsSync(...args) })) + +import { buildYtdlpCommand } from '../ytdlp' + +function setPlatform(platform: NodeJS.Platform) { + Object.defineProperty(process, 'platform', { value: platform }) +} + +describe('buildYtdlpCommand', () => { + const originalPlatform = process.platform + + afterEach(() => { + setPlatform(originalPlatform) + mockExistsSync.mockReset() + }) + + it('runs the bundled binary directly on Linux', () => { + setPlatform('linux') + mockExistsSync.mockReturnValue(true) + const { command, args } = buildYtdlpCommand(['-J', 'https://x.test']) + expect(command).toContain('yt-dlp') + expect(args).toEqual(['-J', 'https://x.test']) + }) + + it('runs the bundled binary through python on Windows', () => { + setPlatform('win32') + mockExistsSync.mockReturnValue(true) + const { command, args } = buildYtdlpCommand(['-J', 'https://x.test']) + expect(command).toBe('python') + expect(args[0]).toContain('yt-dlp') + expect(args.slice(1)).toEqual(['-J', 'https://x.test']) + }) + + it('runs a system-wide yt-dlp directly on Windows when no bundled binary exists', () => { + setPlatform('win32') + mockExistsSync.mockReturnValue(false) + const { command, args } = buildYtdlpCommand(['-J', 'https://x.test']) + expect(command).toBe('yt-dlp') + expect(args).toEqual(['-J', 'https://x.test']) + }) + + it('runs a system-wide yt-dlp directly on Linux when no bundled binary exists', () => { + setPlatform('linux') + mockExistsSync.mockReturnValue(false) + const { command, args } = buildYtdlpCommand(['-J', 'https://x.test']) + expect(command).toBe('yt-dlp') + expect(args).toEqual(['-J', 'https://x.test']) + }) +}) diff --git a/src/lib/__tests__/ytdlp-probe.test.ts b/src/lib/__tests__/ytdlp-probe.test.ts new file mode 100644 index 0000000..b696ec5 --- /dev/null +++ b/src/lib/__tests__/ytdlp-probe.test.ts @@ -0,0 +1,76 @@ +import { parseProbeOutput, buildProbeArgs, ProbeError } from '../ytdlp-probe' + +function json(obj: unknown): string { + return JSON.stringify(obj) +} + +describe('buildProbeArgs', () => { + it('dumps JSON without downloading and disables playlists', () => { + const args = buildProbeArgs('https://youtube.com/watch?v=abc') + expect(args).toContain('-J') + expect(args).toContain('--skip-download') + expect(args).toContain('--no-playlist') + expect(args[args.length - 1]).toBe('https://youtube.com/watch?v=abc') + }) +}) + +describe('parseProbeOutput', () => { + it('throws a ProbeError on invalid JSON', () => { + expect(() => parseProbeOutput('not json')).toThrow(ProbeError) + }) + + it('detects a video with multiple qualities and subtitles', () => { + const result = parseProbeOutput(json({ + title: 'Some video', + duration: 125, + formats: [ + { height: 360, vcodec: 'avc1' }, + { height: 720, vcodec: 'avc1' }, + { height: 1080, vcodec: 'avc1' }, + { vcodec: 'none', acodec: 'opus' }, + ], + subtitles: { en: [{}] }, + automatic_captions: { fr: [{}], es: [{}] }, + })) + + expect(result.isAudioOnly).toBe(false) + expect(result.durationSeconds).toBe(125) + expect(result.availableQualities).toEqual(['best', '1080p', '720p', '480p', '360p']) + expect(result.subtitleLangs).toEqual(['fr', 'en', 'es']) + }) + + it('caps available qualities to the max height found', () => { + const result = parseProbeOutput(json({ + formats: [ + { height: 360, vcodec: 'avc1' }, + { height: 480, vcodec: 'avc1' }, + ], + })) + expect(result.availableQualities).toEqual(['best', '480p', '360p']) + }) + + it('detects an audio-only source and hides quality options', () => { + const result = parseProbeOutput(json({ + title: 'Some track', + duration: 200, + formats: [ + { vcodec: 'none', acodec: 'mp3' }, + { vcodec: 'none', acodec: 'opus' }, + ], + })) + expect(result.isAudioOnly).toBe(true) + expect(result.availableQualities).toEqual([]) + }) + + it('returns an empty subtitle list when none are available', () => { + const result = parseProbeOutput(json({ + formats: [{ height: 720, vcodec: 'avc1' }], + })) + expect(result.subtitleLangs).toEqual([]) + }) + + it('defaults duration to null when missing', () => { + const result = parseProbeOutput(json({ formats: [] })) + expect(result.durationSeconds).toBeNull() + }) +}) diff --git a/src/lib/__tests__/ytdlp.test.ts b/src/lib/__tests__/ytdlp.test.ts index dfa4530..a137be3 100644 --- a/src/lib/__tests__/ytdlp.test.ts +++ b/src/lib/__tests__/ytdlp.test.ts @@ -6,6 +6,10 @@ const base = { format: 'mp4', quality: 'best', subtitles: false, + subtitleLangs: null, + clipStart: null, + clipEnd: null, + audioQuality: null, extraArgs: null, } @@ -22,23 +26,89 @@ describe('buildYtdlpArgs', () => { expect(args[idx + 1]).toContain('%(ext)s') }) - it('adds format filter when quality is not "best"', () => { - const args = buildYtdlpArgs({ ...base, quality: '1080p' }) - expect(args).toContain('-f') + it('always sets --merge-output-format to the chosen video format', () => { + const args = buildYtdlpArgs(base) + const idx = args.indexOf('--merge-output-format') + expect(idx).toBeGreaterThan(-1) + expect(args[idx + 1]).toBe('mp4') }) - it('does not add -f flag when quality is "best"', () => { - expect(buildYtdlpArgs(base)).not.toContain('-f') + it('adds a height filter to -f when quality is not "best"', () => { + const args = buildYtdlpArgs({ ...base, quality: '1080p' }) + const idx = args.indexOf('-f') + expect(idx).toBeGreaterThan(-1) + expect(args[idx + 1]).toContain('height<=?1080') + }) + + it('does not add a height filter when quality is "best"', () => { + const args = buildYtdlpArgs(base) + const idx = args.indexOf('-f') + expect(args[idx + 1]).not.toContain('height<=?') + }) + + it('extracts audio with -x and --audio-format when format is mp3', () => { + const args = buildYtdlpArgs({ ...base, format: 'mp3' }) + expect(args).toContain('-x') + const idx = args.indexOf('--audio-format') + expect(args[idx + 1]).toBe('mp3') + expect(args).not.toContain('--merge-output-format') + }) + + it('sets --audio-quality to 0 for best mp3 quality', () => { + const args = buildYtdlpArgs({ ...base, format: 'mp3', audioQuality: 'best' }) + const idx = args.indexOf('--audio-quality') + expect(args[idx + 1]).toBe('0') + }) + + it('sets --audio-quality to a bitrate when a specific mp3 quality is given', () => { + const args = buildYtdlpArgs({ ...base, format: 'mp3', audioQuality: '192' }) + const idx = args.indexOf('--audio-quality') + expect(args[idx + 1]).toBe('192K') }) it('adds subtitle flags when subtitles is true', () => { const args = buildYtdlpArgs({ ...base, subtitles: true }) - expect(args).toContain('--write-sub') + expect(args).toContain('--write-subs') + expect(args).toContain('--write-auto-subs') expect(args).toContain('--sub-lang') }) it('does not add subtitle flags when subtitles is false', () => { - expect(buildYtdlpArgs(base)).not.toContain('--write-sub') + expect(buildYtdlpArgs(base)).not.toContain('--write-subs') + }) + + it('defaults subtitle languages to fr,en when none are given', () => { + const args = buildYtdlpArgs({ ...base, subtitles: true }) + const idx = args.indexOf('--sub-lang') + expect(args[idx + 1]).toBe('fr,en') + }) + + it('uses the given subtitle languages when provided', () => { + const args = buildYtdlpArgs({ ...base, subtitles: true, subtitleLangs: ['es', 'it'] }) + const idx = args.indexOf('--sub-lang') + expect(args[idx + 1]).toBe('es,it') + }) + + it('adds --download-sections when a clip range is given', () => { + const args = buildYtdlpArgs({ ...base, clipStart: 10, clipEnd: 30 }) + const idx = args.indexOf('--download-sections') + expect(args[idx + 1]).toBe('*10-30') + }) + + it('defaults clip start to 0 when only clipEnd is given', () => { + const args = buildYtdlpArgs({ ...base, clipEnd: 30 }) + const idx = args.indexOf('--download-sections') + expect(args[idx + 1]).toBe('*0-30') + }) + + it('leaves the clip end open when only clipStart is given', () => { + const args = buildYtdlpArgs({ ...base, clipStart: 10 }) + const idx = args.indexOf('--download-sections') + expect(args[idx + 1]).toBe('*10-') + }) + + it('does not add --download-sections when no clip range is given', () => { + expect(buildYtdlpArgs(base)).not.toContain('--download-sections') }) it('appends extra args from a JSON array string', () => { diff --git a/src/lib/rate-limit.ts b/src/lib/rate-limit.ts index 0dc17e6..fdb08b1 100644 --- a/src/lib/rate-limit.ts +++ b/src/lib/rate-limit.ts @@ -1,19 +1,34 @@ import { config } from '../../config/app.config' type Entry = { count: number; resetAt: number } -const store = new Map() -export function isRateLimited(ip: string): boolean { - const now = Date.now() - const entry = store.get(ip) +export function createRateLimiter(max: number, windowMs: number) { + const store = new Map() - if (!entry || now > entry.resetAt) { - store.set(ip, { count: 1, resetAt: now + config.RATE_LIMIT_WINDOW_MS }) - return false + return { + isLimited(ip: string): boolean { + const now = Date.now() + const entry = store.get(ip) + + if (!entry || now > entry.resetAt) { + store.set(ip, { count: 1, resetAt: now + windowMs }) + return false + } + + if (entry.count >= max) return true + + entry.count++ + return false + }, } - - if (entry.count >= config.RATE_LIMIT_MAX) return true - - entry.count++ - return false } + +export const downloadRateLimiter = createRateLimiter( + config.RATE_LIMIT_MAX, + config.RATE_LIMIT_WINDOW_MS +) + +export const probeRateLimiter = createRateLimiter( + config.PROBE_RATE_LIMIT_MAX, + config.PROBE_RATE_LIMIT_WINDOW_MS +) diff --git a/src/lib/request-ip.ts b/src/lib/request-ip.ts new file mode 100644 index 0000000..8986835 --- /dev/null +++ b/src/lib/request-ip.ts @@ -0,0 +1,5 @@ +import { NextRequest } from 'next/server' + +export function getClientIp(req: NextRequest): string { + return req.headers.get('x-forwarded-for')?.split(',')[0].trim() ?? '0.0.0.0' +} diff --git a/src/lib/ytdlp-options.ts b/src/lib/ytdlp-options.ts new file mode 100644 index 0000000..7e0a90d --- /dev/null +++ b/src/lib/ytdlp-options.ts @@ -0,0 +1,3 @@ +export const FORMATS = ['mp4', 'mp3', 'webm', 'mkv'] as const +export const QUALITIES = ['best', '1080p', '720p', '480p', '360p'] as const +export const AUDIO_QUALITIES = ['best', '192', '128'] as const diff --git a/src/lib/ytdlp-probe.ts b/src/lib/ytdlp-probe.ts new file mode 100644 index 0000000..73a8d61 --- /dev/null +++ b/src/lib/ytdlp-probe.ts @@ -0,0 +1,125 @@ +import { spawn } from 'child_process' +import { config } from '../../config/app.config' +import { buildYtdlpCommand } from './ytdlp' +import { QUALITIES } from './ytdlp-options' + +export type ProbeCapabilities = { + title: string + durationSeconds: number | null + isAudioOnly: boolean + availableQualities: string[] + subtitleLangs: string[] +} + +type YtdlpFormat = { + height?: number | null + vcodec?: string | null +} + +type YtdlpProbeJson = { + title?: string + duration?: number | null + formats?: YtdlpFormat[] + subtitles?: Record + automatic_captions?: Record +} + +export class ProbeError extends Error {} + +export function buildProbeArgs(url: string): string[] { + return ['--no-warnings', '--skip-download', '--no-playlist', '-J', url] +} + +const LANG_PRIORITY = ['fr', 'en'] + +export function parseProbeOutput(stdout: string): ProbeCapabilities { + let json: YtdlpProbeJson + try { + json = JSON.parse(stdout) + } catch { + throw new ProbeError('yt-dlp returned invalid JSON') + } + + const formats = json.formats ?? [] + const videoFormats = formats.filter((f) => f.vcodec && f.vcodec !== 'none') + const isAudioOnly = formats.length > 0 && videoFormats.length === 0 + + const heights = videoFormats + .map((f) => f.height) + .filter((h): h is number => typeof h === 'number') + const maxHeight = heights.length ? Math.max(...heights) : null + + const availableQualities = isAudioOnly + ? [] + : QUALITIES.filter( + (q) => q === 'best' || (maxHeight != null && Number(q.replace('p', '')) <= maxHeight) + ) + + const langs = new Set([ + ...Object.keys(json.subtitles ?? {}), + ...Object.keys(json.automatic_captions ?? {}), + ]) + const subtitleLangs = [ + ...LANG_PRIORITY.filter((l) => langs.has(l)), + ...[...langs].filter((l) => !LANG_PRIORITY.includes(l)).sort(), + ] + + return { + title: json.title ?? '', + durationSeconds: typeof json.duration === 'number' ? json.duration : null, + isAudioOnly, + availableQualities, + subtitleLangs, + } +} + +export function runYtdlpProbe( + url: string, + timeoutMs: number = config.PROBE_TIMEOUT_MS +): Promise { + return new Promise((resolve, reject) => { + const { command, args } = buildYtdlpCommand(buildProbeArgs(url)) + const proc = spawn(command, args) + let stdout = '' + let stderr = '' + let settled = false + + const timer = setTimeout(() => { + if (settled) return + settled = true + proc.kill('SIGKILL') + reject(new ProbeError('yt-dlp probe timed out')) + }, timeoutMs) + + proc.stdout.on('data', (chunk: Buffer) => { + stdout += chunk.toString() + }) + proc.stderr.on('data', (chunk: Buffer) => { + stderr += chunk.toString() + }) + + proc.on('error', (err) => { + if (settled) return + settled = true + clearTimeout(timer) + reject(new ProbeError(err.message)) + }) + + proc.on('close', (code) => { + if (settled) return + settled = true + clearTimeout(timer) + + if (code !== 0) { + reject(new ProbeError(stderr || `yt-dlp exited with code ${code}`)) + return + } + + try { + resolve(parseProbeOutput(stdout)) + } catch (err) { + reject(err) + } + }) + }) +} diff --git a/src/lib/ytdlp.ts b/src/lib/ytdlp.ts index d40a5d9..1689d1e 100644 --- a/src/lib/ytdlp.ts +++ b/src/lib/ytdlp.ts @@ -8,11 +8,49 @@ export type YtdlpParams = { format: string quality: string subtitles: boolean + subtitleLangs: string[] | null + clipStart: number | null + clipEnd: number | null + audioQuality: string | null extraArgs: string | null } +export function resolveYtdlpBin(): string { + const local = path.join(config.BIN_DIR, 'yt-dlp') + return existsSync(local) ? local : 'yt-dlp' +} + +export type YtdlpCommand = { command: string; args: string[] } + +// The bundled bin/yt-dlp is a Python zipapp relying on a `#!/usr/bin/env python3` +// shebang: Linux (o2switch prod) execs it natively, but Windows' spawn() has no +// shebang support and fails with ENOENT, so it must be run through `python` there. +// A system-wide `yt-dlp` (PATH fallback) already has a proper platform launcher +// on both OSes and is always run directly. +export function buildYtdlpCommand(args: string[]): YtdlpCommand { + const bin = resolveYtdlpBin() + const isBundledScript = bin !== 'yt-dlp' + + if (process.platform === 'win32' && isBundledScript) { + return { command: 'python', args: [bin, ...args] } + } + + return { command: bin, args } +} + export function buildYtdlpArgs(params: YtdlpParams): string[] { - const { url, uuid, format, quality, subtitles, extraArgs } = params + const { + url, + uuid, + format, + quality, + subtitles, + subtitleLangs, + clipStart, + clipEnd, + audioQuality, + extraArgs, + } = params const outputTemplate = path.join(config.STORAGE_PATH, `${uuid}.%(ext)s`) const args: string[] = ['--no-playlist', '-o', outputTemplate] @@ -22,13 +60,22 @@ export function buildYtdlpArgs(params: YtdlpParams): string[] { args.push('--ffmpeg-location', ffmpegBin) } - if (quality !== 'best') { - const height = quality.replace('p', '') - args.push('-f', `${format}[height<=?${height}]+bestaudio/best[height<=?${height}]`) + if (format === 'mp3') { + args.push('-x', '--audio-format', 'mp3') + args.push('--audio-quality', audioQuality && audioQuality !== 'best' ? `${audioQuality}K` : '0') + } else { + const heightFilter = quality !== 'best' ? `[height<=?${quality.replace('p', '')}]` : '' + args.push('-f', `bestvideo${heightFilter}+bestaudio/best${heightFilter}`) + args.push('--merge-output-format', format) } if (subtitles) { - args.push('--write-sub', '--sub-lang', 'fr,en') + const langs = subtitleLangs?.length ? subtitleLangs.join(',') : 'fr,en' + args.push('--write-subs', '--write-auto-subs', '--sub-lang', langs) + } + + if (clipStart != null || clipEnd != null) { + args.push('--download-sections', `*${clipStart ?? 0}-${clipEnd ?? ''}`) } if (extraArgs) { diff --git a/worker/__tests__/processor.test.ts b/worker/__tests__/processor.test.ts index 57ec5f2..242b10b 100644 --- a/worker/__tests__/processor.test.ts +++ b/worker/__tests__/processor.test.ts @@ -29,6 +29,10 @@ const baseDownload = { format: 'mp4', quality: 'best', subtitles: false, + subtitleLangs: null, + clipStart: null, + clipEnd: null, + audioQuality: null, extraArgs: null, } diff --git a/worker/processor.ts b/worker/processor.ts index 27196d4..e76d647 100644 --- a/worker/processor.ts +++ b/worker/processor.ts @@ -1,16 +1,11 @@ import { spawn } from 'child_process' -import { existsSync, readdirSync, statSync } from 'fs' +import { readdirSync, statSync } from 'fs' import path from 'path' import { prisma } from '@/lib/prisma' -import { buildYtdlpArgs } from '@/lib/ytdlp' +import { buildYtdlpArgs, buildYtdlpCommand } from '@/lib/ytdlp' import { createToken } from '@/lib/token' import { config } from '../config/app.config' -function resolveYtdlpBin(): string { - const local = path.join(config.BIN_DIR, 'yt-dlp') - return existsSync(local) ? local : 'yt-dlp' -} - export async function processDownload(downloadId: string): Promise { const download = await prisma.download.findUnique({ where: { id: downloadId } }) if (!download) return @@ -21,13 +16,18 @@ export async function processDownload(downloadId: string): Promise { format: download.format, quality: download.quality, subtitles: download.subtitles, + subtitleLangs: download.subtitleLangs ? download.subtitleLangs.split(',') : null, + clipStart: download.clipStart, + clipEnd: download.clipEnd, + audioQuality: download.audioQuality, extraArgs: download.extraArgs, }) let stderr = '' + const { command, args: spawnArgs } = buildYtdlpCommand(args) await new Promise((resolve) => { - const proc = spawn(resolveYtdlpBin(), args) + const proc = spawn(command, spawnArgs) proc.stderr.on('data', (chunk: Buffer) => { stderr += chunk.toString() }) proc.on('close', async (code) => {