Ombrora/convert
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add auto-increment id, size/duration tracking, and soft cleanup to conversion_jobs

Browse Source 
Rename the old CHAR(36) id to uuid (still used for public URLs and file
naming) and add a real auto-increment id as the primary key. Track
input/output file size and conversion duration per job. Cleanup no
longer deletes rows; it marks cleaned_at and skips already-cleaned
expired jobs on later runs.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
anthony
2026-07-30 22:06:32 +02:00
co-authored by Claude Sonnet 5
parent 24f7ab2625
commit ff5a257a0e
10 changed files with 208 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/schema.sql
+7 -1
View File
@@ -1,5 +1,6 @@
CREATE TABLE IF NOT EXISTS conversion_jobs ( CREATE TABLE IF NOT EXISTS conversion_jobs (
id CHAR(36) NOT NULL PRIMARY KEY, id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
uuid CHAR(36) NOT NULL,
status ENUM('pending', 'processing', 'done', 'failed') NOT NULL DEFAULT 'pending', status ENUM('pending', 'processing', 'done', 'failed') NOT NULL DEFAULT 'pending',
family VARCHAR(32) NOT NULL, family VARCHAR(32) NOT NULL,
source_format VARCHAR(16) NOT NULL, source_format VARCHAR(16) NOT NULL,
@@ -9,11 +10,16 @@ CREATE TABLE IF NOT EXISTS conversion_jobs (
output_path VARCHAR(255) NULL, output_path VARCHAR(255) NULL,
input_mime_type VARCHAR(128) NOT NULL, input_mime_type VARCHAR(128) NOT NULL,
output_mime_type VARCHAR(128) NULL, output_mime_type VARCHAR(128) NULL,
input_size_bytes INT UNSIGNED NOT NULL,
output_size_bytes INT UNSIGNED NULL,
conversion_duration_seconds DECIMAL(10,3) NULL,
error_message VARCHAR(255) NULL, error_message VARCHAR(255) NULL,
error_log TEXT NULL, error_log TEXT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
expires_at DATETIME NOT NULL, expires_at DATETIME NOT NULL,
cleaned_at DATETIME NULL DEFAULT NULL,
UNIQUE KEY uniq_uuid (uuid),
INDEX idx_status (status), INDEX idx_status (status),
INDEX idx_expires_at (expires_at) INDEX idx_expires_at (expires_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
src/app.js
+9 -8
View File
@@ -10,7 +10,7 @@ import { registerImageToPdfConverter } from './converters/imageToPdf.js';
import { registerDocumentConverters } from './converters/document.js'; import { registerDocumentConverters } from './converters/document.js';
import { resolveInputFormat } from './mime.js'; import { resolveInputFormat } from './mime.js';
import { deleteIfExists } from './storage.js'; import { deleteIfExists } from './storage.js';
import { createJob, getJobById } from './jobs/jobRepository.js'; import { createJob, getJobByUuid } from './jobs/jobRepository.js';
import { outputPath } from './storage.js'; import { outputPath } from './storage.js';
let convertersRegistered = false; let convertersRegistered = false;
@@ -77,7 +77,7 @@ export function createApp(config, pool) {
for (let i = 0; i < req.files.length; i += 1) { for (let i = 0; i < req.files.length; i += 1) {
const file = req.files[i]; const file = req.files[i];
const targetFormat = targetFormats[i]; const targetFormat = targetFormats[i];
const id = path.basename(file.filename, path.extname(file.filename)); const uuid = path.basename(file.filename, path.extname(file.filename));
const sourceFormat = path.extname(file.filename).slice(1).toLowerCase(); const sourceFormat = path.extname(file.filename).slice(1).toLowerCase();
const { mime, valid } = await resolveInputFormat(file.path, sourceFormat); const { mime, valid } = await resolveInputFormat(file.path, sourceFormat);
@@ -99,30 +99,31 @@ export function createApp(config, pool) {
const expiresAt = new Date(Date.now() + config.retentionHours * 3600 * 1000); const expiresAt = new Date(Date.now() + config.retentionHours * 3600 * 1000);
await createJob(pool, { await createJob(pool, {
id, uuid,
family: registryEntry.family, family: registryEntry.family,
sourceFormat, sourceFormat,
targetFormat, targetFormat,
originalFilename: file.originalname, originalFilename: file.originalname,
inputPath: file.filename, inputPath: file.filename,
inputMimeType: mime, inputMimeType: mime,
inputSizeBytes: file.size,
expiresAt, expiresAt,
}); });
results.push({ file: file.originalname, id, status: 'pending' }); results.push({ file: file.originalname, id: uuid, status: 'pending' });
} }
res.status(201).json({ jobs: results }); res.status(201).json({ jobs: results });
}); });
app.get('/api/jobs/:id', async (req, res) => { app.get('/api/jobs/:id', async (req, res) => {
const job = await getJobById(pool, req.params.id); const job = await getJobByUuid(pool, req.params.id);
if (!job) { if (!job) {
return res.status(404).json({ error: 'Job not found' }); return res.status(404).json({ error: 'Job not found' });
} }
res.json({ res.json({
id: job.id, id: job.uuid,
status: job.status, status: job.status,
originalFilename: job.originalFilename, originalFilename: job.originalFilename,
sourceFormat: job.sourceFormat, sourceFormat: job.sourceFormat,
@@ -138,7 +139,7 @@ export function createApp(config, pool) {
} }
app.get('/api/jobs/:id/download', async (req, res) => { app.get('/api/jobs/:id/download', async (req, res) => {
const job = await getJobById(pool, req.params.id); const job = await getJobByUuid(pool, req.params.id);
if (!job) { if (!job) {
return res.status(404).json({ error: 'Job not found' }); return res.status(404).json({ error: 'Job not found' });
} }
@@ -146,7 +147,7 @@ export function createApp(config, pool) {
return res.status(409).json({ error: `Job is not ready yet (status: ${job.status})` }); return res.status(409).json({ error: `Job is not ready yet (status: ${job.status})` });
} }
const filePath = outputPath(config, job.id, job.targetFormat); const filePath = outputPath(config, job.uuid, job.targetFormat);
const downloadFilename = `${path.parse(job.originalFilename).name}.${job.targetFormat}`; const downloadFilename = `${path.parse(job.originalFilename).name}.${job.targetFormat}`;
res.set('Content-Type', job.outputMimeType); res.set('Content-Type', job.outputMimeType);
res.set('Content-Disposition', contentDispositionHeader(downloadFilename)); res.set('Content-Disposition', contentDispositionHeader(downloadFilename));
src/cleanup.js
+4 -4
View File
@@ -2,15 +2,15 @@ import { pathToFileURL } from 'node:url';
import { loadConfig } from './config.js'; import { loadConfig } from './config.js';
import { getPool, closePool } from './db.js'; import { getPool, closePool } from './db.js';
import { uploadPath, outputPath, deleteIfExists } from './storage.js'; import { uploadPath, outputPath, deleteIfExists } from './storage.js';
import { findExpiredJobs, deleteJob } from './jobs/jobRepository.js'; import { findExpiredJobs, markCleaned } from './jobs/jobRepository.js';
export async function runCleanup(pool, config) { export async function runCleanup(pool, config) {
const expiredJobs = await findExpiredJobs(pool); const expiredJobs = await findExpiredJobs(pool);
for (const job of expiredJobs) { for (const job of expiredJobs) {
await deleteIfExists(uploadPath(config, job.id, job.sourceFormat)); await deleteIfExists(uploadPath(config, job.uuid, job.sourceFormat));
await deleteIfExists(outputPath(config, job.id, job.targetFormat)); await deleteIfExists(outputPath(config, job.uuid, job.targetFormat));
await deleteJob(pool, job.id); await markCleaned(pool, job.id);
} }
return expiredJobs.length; return expiredJobs.length;
src/jobs/jobRepository.js
+29 -18
View File
@@ -2,6 +2,7 @@ function toCamelJob(row) {
if (!row) return null; if (!row) return null;
return { return {
id: row.id, id: row.id,
uuid: row.uuid,
status: row.status, status: row.status,
family: row.family, family: row.family,
sourceFormat: row.source_format, sourceFormat: row.source_format,
@@ -11,38 +12,44 @@ function toCamelJob(row) {
outputPath: row.output_path, outputPath: row.output_path,
inputMimeType: row.input_mime_type, inputMimeType: row.input_mime_type,
outputMimeType: row.output_mime_type, outputMimeType: row.output_mime_type,
inputSizeBytes: row.input_size_bytes,
outputSizeBytes: row.output_size_bytes,
conversionDurationSeconds: row.conversion_duration_seconds,
errorMessage: row.error_message, errorMessage: row.error_message,
createdAt: row.created_at, createdAt: row.created_at,
updatedAt: row.updated_at, updatedAt: row.updated_at,
expiresAt: row.expires_at, expiresAt: row.expires_at,
cleanedAt: row.cleaned_at,
}; };
} }
export async function createJob(pool, job) { export async function createJob(pool, job) {
await pool.query( await pool.query(
`INSERT INTO conversion_jobs `INSERT INTO conversion_jobs
(id, status, family, source_format, target_format, original_filename, input_path, input_mime_type, expires_at) (uuid, status, family, source_format, target_format, original_filename, input_path, input_mime_type, input_size_bytes, expires_at)
VALUES (?, 'pending', ?, ?, ?, ?, ?, ?, ?)`, VALUES (?, 'pending', ?, ?, ?, ?, ?, ?, ?, ?)`,
[ [
job.id, job.uuid,
job.family, job.family,
job.sourceFormat, job.sourceFormat,
job.targetFormat, job.targetFormat,
job.originalFilename, job.originalFilename,
job.inputPath, job.inputPath,
job.inputMimeType, job.inputMimeType,
job.inputSizeBytes,
job.expiresAt, job.expiresAt,
] ]
); );
} }
export async function getJobById(pool, id) { export async function getJobByUuid(pool, uuid) {
const rows = await pool.query( const rows = await pool.query(
`SELECT id, status, family, source_format, target_format, original_filename, `SELECT id, uuid, status, family, source_format, target_format, original_filename,
input_path, output_path, input_mime_type, output_mime_type, input_path, output_path, input_mime_type, output_mime_type,
error_message, created_at, updated_at, expires_at input_size_bytes, output_size_bytes, conversion_duration_seconds,
FROM conversion_jobs WHERE id = ?`, error_message, created_at, updated_at, expires_at, cleaned_at
[id] FROM conversion_jobs WHERE uuid = ?`,
[uuid]
); );
return toCamelJob(rows[0]); return toCamelJob(rows[0]);
} }
@@ -56,10 +63,12 @@ export async function markProcessing(pool, id) {
await pool.query("UPDATE conversion_jobs SET status = 'processing' WHERE id = ?", [id]); await pool.query("UPDATE conversion_jobs SET status = 'processing' WHERE id = ?", [id]);
} }
export async function markDone(pool, id, { outputPath, outputMimeType }) { export async function markDone(pool, id, { outputPath, outputMimeType, outputSizeBytes, conversionDurationSeconds }) {
await pool.query( await pool.query(
"UPDATE conversion_jobs SET status = 'done', output_path = ?, output_mime_type = ? WHERE id = ?", `UPDATE conversion_jobs
[outputPath, outputMimeType, id] SET status = 'done', output_path = ?, output_mime_type = ?, output_size_bytes = ?, conversion_duration_seconds = ?
WHERE id = ?`,
[outputPath, outputMimeType, outputSizeBytes, conversionDurationSeconds, id]
); );
} }
@@ -72,9 +81,10 @@ export async function markFailed(pool, id, { errorMessage, errorLog }) {
export async function findPendingJobs(pool, limit) { export async function findPendingJobs(pool, limit) {
const rows = await pool.query( const rows = await pool.query(
`SELECT id, status, family, source_format, target_format, original_filename, `SELECT id, uuid, status, family, source_format, target_format, original_filename,
input_path, output_path, input_mime_type, output_mime_type, input_path, output_path, input_mime_type, output_mime_type,
error_message, created_at, updated_at, expires_at input_size_bytes, output_size_bytes, conversion_duration_seconds,
error_message, created_at, updated_at, expires_at, cleaned_at
FROM conversion_jobs WHERE status = 'pending' ORDER BY created_at ASC LIMIT ?`, FROM conversion_jobs WHERE status = 'pending' ORDER BY created_at ASC LIMIT ?`,
[limit] [limit]
); );
@@ -83,14 +93,15 @@ export async function findPendingJobs(pool, limit) {
export async function findExpiredJobs(pool) { export async function findExpiredJobs(pool) {
const rows = await pool.query( const rows = await pool.query(
`SELECT id, status, family, source_format, target_format, original_filename, `SELECT id, uuid, status, family, source_format, target_format, original_filename,
input_path, output_path, input_mime_type, output_mime_type, input_path, output_path, input_mime_type, output_mime_type,
error_message, created_at, updated_at, expires_at input_size_bytes, output_size_bytes, conversion_duration_seconds,
FROM conversion_jobs WHERE expires_at < NOW()` error_message, created_at, updated_at, expires_at, cleaned_at
FROM conversion_jobs WHERE expires_at < NOW() AND cleaned_at IS NULL`
); );
return rows.map(toCamelJob); return rows.map(toCamelJob);
} }
export async function deleteJob(pool, id) { export async function markCleaned(pool, id) {
await pool.query('DELETE FROM conversion_jobs WHERE id = ?', [id]); await pool.query('UPDATE conversion_jobs SET cleaned_at = NOW() WHERE id = ?', [id]);
} }
src/worker.js
+9 -3
View File
@@ -1,4 +1,5 @@
import { pathToFileURL } from 'node:url'; import { pathToFileURL } from 'node:url';
import fs from 'node:fs/promises';
import { loadConfig } from './config.js'; import { loadConfig } from './config.js';
import { getPool } from './db.js'; import { getPool } from './db.js';
import { ensureStorageDirs, uploadPath, outputPath, deleteIfExists } from './storage.js'; import { ensureStorageDirs, uploadPath, outputPath, deleteIfExists } from './storage.js';
@@ -22,8 +23,8 @@ function withTimeout(promise, ms) {
async function processJob(pool, config, job) { async function processJob(pool, config, job) {
await markProcessing(pool, job.id); await markProcessing(pool, job.id);
const inputFilePath = uploadPath(config, job.id, job.sourceFormat); const inputFilePath = uploadPath(config, job.uuid, job.sourceFormat);
const outputFilePath = outputPath(config, job.id, job.targetFormat); const outputFilePath = outputPath(config, job.uuid, job.targetFormat);
try { try {
const entry = resolveConverter(job.sourceFormat, job.targetFormat); const entry = resolveConverter(job.sourceFormat, job.targetFormat);
@@ -31,11 +32,16 @@ async function processJob(pool, config, job) {
throw new Error(`No converter registered for ${job.sourceFormat} -> ${job.targetFormat}`); throw new Error(`No converter registered for ${job.sourceFormat} -> ${job.targetFormat}`);
} }
const startedAt = Date.now();
await withTimeout(entry.convert(inputFilePath, outputFilePath), JOB_TIMEOUT_MS); await withTimeout(entry.convert(inputFilePath, outputFilePath), JOB_TIMEOUT_MS);
const conversionDurationSeconds = (Date.now() - startedAt) / 1000;
const { size: outputSizeBytes } = await fs.stat(outputFilePath);
await markDone(pool, job.id, { await markDone(pool, job.id, {
outputPath: `${job.id}.${job.targetFormat}`, outputPath: `${job.uuid}.${job.targetFormat}`,
outputMimeType: outputMimeType(job.targetFormat), outputMimeType: outputMimeType(job.targetFormat),
outputSizeBytes,
conversionDurationSeconds,
}); });
} catch (error) { } catch (error) {
await deleteIfExists(outputFilePath); await deleteIfExists(outputFilePath);
test/api/jobStatus.test.js
+23 -15
View File
@@ -6,7 +6,7 @@ import os from 'node:os';
import { createApp } from '../../src/app.js'; import { createApp } from '../../src/app.js';
import { getPool, closePool } from '../../src/db.js'; import { getPool, closePool } from '../../src/db.js';
import { loadConfig } from '../../src/config.js'; import { loadConfig } from '../../src/config.js';
import { createJob, markDone } from '../../src/jobs/jobRepository.js'; import { createJob, getJobByUuid, markDone } from '../../src/jobs/jobRepository.js';
import { ensureStorageDirs, outputPath } from '../../src/storage.js'; import { ensureStorageDirs, outputPath } from '../../src/storage.js';
let app; let app;
@@ -29,27 +29,29 @@ beforeEach(async () => {
await pool.query('DELETE FROM conversion_jobs'); await pool.query('DELETE FROM conversion_jobs');
}); });
function baseJob(id) { function baseJob(uuid) {
return { return {
id, uuid,
family: 'image', family: 'image',
sourceFormat: 'png', sourceFormat: 'png',
targetFormat: 'webp', targetFormat: 'webp',
originalFilename: 'holiday photo.png', originalFilename: 'holiday photo.png',
inputPath: `${id}.png`, inputPath: `${uuid}.png`,
inputMimeType: 'image/png', inputMimeType: 'image/png',
inputSizeBytes: 11,
expiresAt: new Date(Date.now() + 3600 * 1000), expiresAt: new Date(Date.now() + 3600 * 1000),
}; };
} }
describe('GET /api/jobs/:id', () => { describe('GET /api/jobs/:id', () => {
it('returns job status without the error log field', async () => { it('returns job status without the error log field', async () => {
const id = '66666666-6666-4666-8666-666666666666'; const uuid = '66666666-6666-4666-8666-666666666666';
await createJob(pool, baseJob(id)); await createJob(pool, baseJob(uuid));
const response = await request(app).get(`/api/jobs/${id}`); const response = await request(app).get(`/api/jobs/${uuid}`);
expect(response.status).toBe(200); expect(response.status).toBe(200);
expect(response.body.id).toBe(uuid);
expect(response.body.status).toBe('pending'); expect(response.body.status).toBe('pending');
expect(response.body.originalFilename).toBe('holiday photo.png'); expect(response.body.originalFilename).toBe('holiday photo.png');
expect(response.body.errorLog).toBeUndefined(); expect(response.body.errorLog).toBeUndefined();
@@ -63,13 +65,19 @@ describe('GET /api/jobs/:id', () => {
describe('GET /api/jobs/:id/download', () => { describe('GET /api/jobs/:id/download', () => {
it('streams the converted file with correct headers once done', async () => { it('streams the converted file with correct headers once done', async () => {
const id = '77777777-7777-4777-8777-777777777777'; const uuid = '77777777-7777-4777-8777-777777777777';
await createJob(pool, baseJob(id)); await createJob(pool, baseJob(uuid));
const filePath = outputPath(config, id, 'webp'); const filePath = outputPath(config, uuid, 'webp');
await fs.writeFile(filePath, Buffer.from('fake webp bytes')); await fs.writeFile(filePath, Buffer.from('fake webp bytes'));
await markDone(pool, id, { outputPath: `${id}.webp`, outputMimeType: 'image/webp' }); const created = await getJobByUuid(pool, uuid);
await markDone(pool, created.id, {
outputPath: `${uuid}.webp`,
outputMimeType: 'image/webp',
outputSizeBytes: 16,
conversionDurationSeconds: 0.2,
});
const response = await request(app).get(`/api/jobs/${id}/download`); const response = await request(app).get(`/api/jobs/${uuid}/download`);
expect(response.status).toBe(200); expect(response.status).toBe(200);
expect(response.headers['content-type']).toBe('image/webp'); expect(response.headers['content-type']).toBe('image/webp');
@@ -78,10 +86,10 @@ describe('GET /api/jobs/:id/download', () => {
}); });
it('returns 409 when the job is not done yet', async () => { it('returns 409 when the job is not done yet', async () => {
const id = '88888888-8888-4888-8888-888888888888'; const uuid = '88888888-8888-4888-8888-888888888888';
await createJob(pool, baseJob(id)); await createJob(pool, baseJob(uuid));
const response = await request(app).get(`/api/jobs/${id}/download`); const response = await request(app).get(`/api/jobs/${uuid}/download`);
expect(response.status).toBe(409); expect(response.status).toBe(409);
}); });
test/api/jobs.test.js
+3 -2
View File
@@ -6,7 +6,7 @@ import os from 'node:os';
import { createApp } from '../../src/app.js'; import { createApp } from '../../src/app.js';
import { getPool, closePool } from '../../src/db.js'; import { getPool, closePool } from '../../src/db.js';
import { loadConfig } from '../../src/config.js'; import { loadConfig } from '../../src/config.js';
import { getJobById } from '../../src/jobs/jobRepository.js'; import { getJobByUuid } from '../../src/jobs/jobRepository.js';
import { ensureStorageDirs } from '../../src/storage.js'; import { ensureStorageDirs } from '../../src/storage.js';
let app; let app;
@@ -60,12 +60,13 @@ describe('POST /api/jobs', () => {
expect(response.body.jobs[0].status).toBe('pending'); expect(response.body.jobs[0].status).toBe('pending');
expect(response.body.jobs[0].file).toBe('photo.png'); expect(response.body.jobs[0].file).toBe('photo.png');
const job = await getJobById(pool, response.body.jobs[0].id); const job = await getJobByUuid(pool, response.body.jobs[0].id);
expect(job.status).toBe('pending'); expect(job.status).toBe('pending');
expect(job.sourceFormat).toBe('png'); expect(job.sourceFormat).toBe('png');
expect(job.targetFormat).toBe('webp'); expect(job.targetFormat).toBe('webp');
expect(job.originalFilename).toBe('photo.png'); expect(job.originalFilename).toBe('photo.png');
expect(job.inputMimeType).toBe('image/png'); expect(job.inputMimeType).toBe('image/png');
expect(job.inputSizeBytes).toBeGreaterThan(0);
}); });
it('rejects a file whose content does not match its extension, without failing the whole batch', async () => { it('rejects a file whose content does not match its extension, without failing the whole batch', async () => {
test/cleanup.test.js
+63 -27
View File
@@ -5,7 +5,7 @@ import os from 'node:os';
import { getPool, closePool } from '../src/db.js'; import { getPool, closePool } from '../src/db.js';
import { loadConfig } from '../src/config.js'; import { loadConfig } from '../src/config.js';
import { ensureStorageDirs, uploadPath, outputPath } from '../src/storage.js'; import { ensureStorageDirs, uploadPath, outputPath } from '../src/storage.js';
import { createJob, markDone, getJobById } from '../src/jobs/jobRepository.js'; import { createJob, markDone, getJobByUuid } from '../src/jobs/jobRepository.js';
import { runCleanup } from '../src/cleanup.js'; import { runCleanup } from '../src/cleanup.js';
let pool; let pool;
@@ -27,66 +27,102 @@ beforeEach(async () => {
}); });
describe('runCleanup', () => { describe('runCleanup', () => {
it('deletes an expired done job, its input file, and its output file', async () => { it('marks an expired done job cleaned and removes its input and output files, keeping the row', async () => {
const id = 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee'; const uuid = 'eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee';
await fs.writeFile(uploadPath(config, id, 'png'), 'input bytes'); await fs.writeFile(uploadPath(config, uuid, 'png'), 'input bytes');
await fs.writeFile(outputPath(config, id, 'webp'), 'output bytes'); await fs.writeFile(outputPath(config, uuid, 'webp'), 'output bytes');
await createJob(pool, { await createJob(pool, {
id, uuid,
family: 'image', family: 'image',
sourceFormat: 'png', sourceFormat: 'png',
targetFormat: 'webp', targetFormat: 'webp',
originalFilename: 'photo.png', originalFilename: 'photo.png',
inputPath: `${id}.png`, inputPath: `${uuid}.png`,
inputMimeType: 'image/png', inputMimeType: 'image/png',
inputSizeBytes: 11,
expiresAt: new Date(Date.now() - 1000), expiresAt: new Date(Date.now() - 1000),
}); });
await markDone(pool, id, { outputPath: `${id}.webp`, outputMimeType: 'image/webp' }); const created = await getJobByUuid(pool, uuid);
await markDone(pool, created.id, {
outputPath: `${uuid}.webp`,
outputMimeType: 'image/webp',
outputSizeBytes: 12,
conversionDurationSeconds: 0.5,
});
const deletedCount = await runCleanup(pool, config); const cleanedCount = await runCleanup(pool, config);
expect(deletedCount).toBe(1); expect(cleanedCount).toBe(1);
expect(await getJobById(pool, id)).toBeNull(); const job = await getJobByUuid(pool, uuid);
await expect(fs.stat(uploadPath(config, id, 'png'))).rejects.toThrow(); expect(job).not.toBeNull();
await expect(fs.stat(outputPath(config, id, 'webp'))).rejects.toThrow(); expect(job.cleanedAt).not.toBeNull();
await expect(fs.stat(uploadPath(config, uuid, 'png'))).rejects.toThrow();
await expect(fs.stat(outputPath(config, uuid, 'webp'))).rejects.toThrow();
}); });
it('deletes an expired pending job (with no output file) without throwing', async () => { it('marks an expired pending job (with no output file) cleaned without throwing', async () => {
const id = 'ffffffff-ffff-4fff-8fff-ffffffffffff'; const uuid = 'ffffffff-ffff-4fff-8fff-ffffffffffff';
await fs.writeFile(uploadPath(config, id, 'png'), 'input bytes'); await fs.writeFile(uploadPath(config, uuid, 'png'), 'input bytes');
await createJob(pool, { await createJob(pool, {
id, uuid,
family: 'image', family: 'image',
sourceFormat: 'png', sourceFormat: 'png',
targetFormat: 'webp', targetFormat: 'webp',
originalFilename: 'photo.png', originalFilename: 'photo.png',
inputPath: `${id}.png`, inputPath: `${uuid}.png`,
inputMimeType: 'image/png', inputMimeType: 'image/png',
inputSizeBytes: 11,
expiresAt: new Date(Date.now() - 1000), expiresAt: new Date(Date.now() - 1000),
}); });
const deletedCount = await runCleanup(pool, config); const cleanedCount = await runCleanup(pool, config);
expect(deletedCount).toBe(1); expect(cleanedCount).toBe(1);
expect(await getJobById(pool, id)).toBeNull(); const job = await getJobByUuid(pool, uuid);
expect(job).not.toBeNull();
expect(job.cleanedAt).not.toBeNull();
}); });
it('leaves non-expired jobs untouched', async () => { it('leaves non-expired jobs untouched', async () => {
const id = '12121212-1212-4212-8212-121212121212'; const uuid = '12121212-1212-4212-8212-121212121212';
await createJob(pool, { await createJob(pool, {
id, uuid,
family: 'image', family: 'image',
sourceFormat: 'png', sourceFormat: 'png',
targetFormat: 'webp', targetFormat: 'webp',
originalFilename: 'photo.png', originalFilename: 'photo.png',
inputPath: `${id}.png`, inputPath: `${uuid}.png`,
inputMimeType: 'image/png', inputMimeType: 'image/png',
inputSizeBytes: 11,
expiresAt: new Date(Date.now() + 3600 * 1000), expiresAt: new Date(Date.now() + 3600 * 1000),
}); });
const deletedCount = await runCleanup(pool, config); const cleanedCount = await runCleanup(pool, config);
expect(deletedCount).toBe(0); expect(cleanedCount).toBe(0);
expect(await getJobById(pool, id)).not.toBeNull(); const job = await getJobByUuid(pool, uuid);
expect(job).not.toBeNull();
expect(job.cleanedAt).toBeNull();
});
it('does not re-process an already cleaned expired job', async () => {
const uuid = '13131313-1313-4313-8313-131313131313';
await createJob(pool, {
uuid,
family: 'image',
sourceFormat: 'png',
targetFormat: 'webp',
originalFilename: 'photo.png',
inputPath: `${uuid}.png`,
inputMimeType: 'image/png',
inputSizeBytes: 11,
expiresAt: new Date(Date.now() - 1000),
});
const firstRun = await runCleanup(pool, config);
const secondRun = await runCleanup(pool, config);
expect(firstRun).toBe(1);
expect(secondRun).toBe(0);
}); });
}); });
test/jobs/jobRepository.test.js
+41 -24
View File
@@ -3,14 +3,14 @@ import { getPool, closePool } from '../../src/db.js';
import { loadConfig } from '../../src/config.js'; import { loadConfig } from '../../src/config.js';
import { import {
createJob, createJob,
getJobById, getJobByUuid,
getJobErrorLog, getJobErrorLog,
markProcessing, markProcessing,
markDone, markDone,
markFailed, markFailed,
findPendingJobs, findPendingJobs,
findExpiredJobs, findExpiredJobs,
deleteJob, markCleaned,
} from '../../src/jobs/jobRepository.js'; } from '../../src/jobs/jobRepository.js';
let pool; let pool;
@@ -29,13 +29,14 @@ beforeEach(async () => {
function baseJob(overrides = {}) { function baseJob(overrides = {}) {
return { return {
id: overrides.id ?? '11111111-1111-4111-8111-111111111111', uuid: overrides.uuid ?? '11111111-1111-4111-8111-111111111111',
family: 'image', family: 'image',
sourceFormat: 'png', sourceFormat: 'png',
targetFormat: 'webp', targetFormat: 'webp',
originalFilename: 'photo.png', originalFilename: 'photo.png',
inputPath: `${overrides.id ?? '11111111-1111-4111-8111-111111111111'}.png`, inputPath: `${overrides.uuid ?? '11111111-1111-4111-8111-111111111111'}.png`,
inputMimeType: 'image/png', inputMimeType: 'image/png',
inputSizeBytes: 1024,
expiresAt: new Date(Date.now() + 3600 * 1000), expiresAt: new Date(Date.now() + 3600 * 1000),
...overrides, ...overrides,
}; };
@@ -45,78 +46,94 @@ describe('jobRepository', () => {
it('creates and retrieves a pending job', async () => { it('creates and retrieves a pending job', async () => {
await createJob(pool, baseJob()); await createJob(pool, baseJob());
const job = await getJobById(pool, '11111111-1111-4111-8111-111111111111'); const job = await getJobByUuid(pool, '11111111-1111-4111-8111-111111111111');
expect(job.id).toEqual(expect.any(Number));
expect(job.uuid).toBe('11111111-1111-4111-8111-111111111111');
expect(job.status).toBe('pending'); expect(job.status).toBe('pending');
expect(job.family).toBe('image'); expect(job.family).toBe('image');
expect(job.sourceFormat).toBe('png'); expect(job.sourceFormat).toBe('png');
expect(job.targetFormat).toBe('webp'); expect(job.targetFormat).toBe('webp');
expect(job.originalFilename).toBe('photo.png'); expect(job.originalFilename).toBe('photo.png');
expect(job.inputMimeType).toBe('image/png'); expect(job.inputMimeType).toBe('image/png');
expect(job.inputSizeBytes).toBe(1024);
expect(job.outputPath).toBeNull(); expect(job.outputPath).toBeNull();
expect(job.outputSizeBytes).toBeNull();
expect(job.conversionDurationSeconds).toBeNull();
expect(job.errorMessage).toBeNull(); expect(job.errorMessage).toBeNull();
expect(job.cleanedAt).toBeNull();
}); });
it('returns null for an unknown id', async () => { it('returns null for an unknown uuid', async () => {
const job = await getJobById(pool, '22222222-2222-4222-8222-222222222222'); const job = await getJobByUuid(pool, '22222222-2222-4222-8222-222222222222');
expect(job).toBeNull(); expect(job).toBeNull();
}); });
it('transitions a job through processing to done', async () => { it('transitions a job through processing to done', async () => {
await createJob(pool, baseJob()); await createJob(pool, baseJob());
await markProcessing(pool, '11111111-1111-4111-8111-111111111111'); await markProcessing(pool, (await getJobByUuid(pool, '11111111-1111-4111-8111-111111111111')).id);
const processing = await getJobById(pool, '11111111-1111-4111-8111-111111111111'); const processing = await getJobByUuid(pool, '11111111-1111-4111-8111-111111111111');
expect(processing.status).toBe('processing'); expect(processing.status).toBe('processing');
await markDone(pool, '11111111-1111-4111-8111-111111111111', { await markDone(pool, processing.id, {
outputPath: '11111111-1111-4111-8111-111111111111.webp', outputPath: '11111111-1111-4111-8111-111111111111.webp',
outputMimeType: 'image/webp', outputMimeType: 'image/webp',
outputSizeBytes: 2048,
conversionDurationSeconds: 1.5,
}); });
const done = await getJobById(pool, '11111111-1111-4111-8111-111111111111'); const done = await getJobByUuid(pool, '11111111-1111-4111-8111-111111111111');
expect(done.status).toBe('done'); expect(done.status).toBe('done');
expect(done.outputPath).toBe('11111111-1111-4111-8111-111111111111.webp'); expect(done.outputPath).toBe('11111111-1111-4111-8111-111111111111.webp');
expect(done.outputMimeType).toBe('image/webp'); expect(done.outputMimeType).toBe('image/webp');
expect(done.outputSizeBytes).toBe(2048);
expect(Number(done.conversionDurationSeconds)).toBe(1.5);
}); });
it('marks a job failed with a short message and a separate detailed log', async () => { it('marks a job failed with a short message and a separate detailed log', async () => {
await createJob(pool, baseJob()); await createJob(pool, baseJob());
const created = await getJobByUuid(pool, '11111111-1111-4111-8111-111111111111');
await markFailed(pool, '11111111-1111-4111-8111-111111111111', { await markFailed(pool, created.id, {
errorMessage: 'Conversion failed, please try again', errorMessage: 'Conversion failed, please try again',
errorLog: 'Error: sharp threw at line 42\n at convert (image.js:10:5)', errorLog: 'Error: sharp threw at line 42\n at convert (image.js:10:5)',
}); });
const job = await getJobById(pool, '11111111-1111-4111-8111-111111111111'); const job = await getJobByUuid(pool, '11111111-1111-4111-8111-111111111111');
expect(job.status).toBe('failed'); expect(job.status).toBe('failed');
expect(job.errorMessage).toBe('Conversion failed, please try again'); expect(job.errorMessage).toBe('Conversion failed, please try again');
expect(job.errorLog).toBeUndefined(); expect(job.errorLog).toBeUndefined();
const errorLog = await getJobErrorLog(pool, '11111111-1111-4111-8111-111111111111'); const errorLog = await getJobErrorLog(pool, created.id);
expect(errorLog).toBe('Error: sharp threw at line 42\n at convert (image.js:10:5)'); expect(errorLog).toBe('Error: sharp threw at line 42\n at convert (image.js:10:5)');
}); });
it('finds pending jobs oldest first, up to a limit', async () => { it('finds pending jobs oldest first, up to a limit', async () => {
await createJob(pool, baseJob({ id: '33333333-3333-4333-8333-333333333333' })); await createJob(pool, baseJob({ uuid: '33333333-3333-4333-8333-333333333333' }));
await createJob(pool, baseJob({ id: '44444444-4444-4444-8444-444444444444' })); await createJob(pool, baseJob({ uuid: '44444444-4444-4444-8444-444444444444' }));
await createJob(pool, baseJob({ id: '55555555-5555-4555-8555-555555555555' })); await createJob(pool, baseJob({ uuid: '55555555-5555-4555-8555-555555555555' }));
const jobs = await findPendingJobs(pool, 2); const jobs = await findPendingJobs(pool, 2);
expect(jobs).toHaveLength(2); expect(jobs).toHaveLength(2);
expect(jobs[0].id).toBe('33333333-3333-4333-8333-333333333333'); expect(jobs[0].uuid).toBe('33333333-3333-4333-8333-333333333333');
expect(jobs[1].id).toBe('44444444-4444-4444-8444-444444444444'); expect(jobs[1].uuid).toBe('44444444-4444-4444-8444-444444444444');
}); });
it('finds expired jobs and allows deleting them', async () => { it('finds expired jobs and allows marking them cleaned without deleting the row', async () => {
await createJob(pool, baseJob({ expiresAt: new Date(Date.now() - 1000) })); await createJob(pool, baseJob({ expiresAt: new Date(Date.now() - 1000) }));
const expired = await findExpiredJobs(pool); const expired = await findExpiredJobs(pool);
expect(expired).toHaveLength(1); expect(expired).toHaveLength(1);
expect(expired[0].id).toBe('11111111-1111-4111-8111-111111111111'); expect(expired[0].uuid).toBe('11111111-1111-4111-8111-111111111111');
await deleteJob(pool, '11111111-1111-4111-8111-111111111111'); await markCleaned(pool, expired[0].id);
const afterDelete = await getJobById(pool, '11111111-1111-4111-8111-111111111111');
expect(afterDelete).toBeNull(); const stillPresent = await getJobByUuid(pool, '11111111-1111-4111-8111-111111111111');
expect(stillPresent).not.toBeNull();
expect(stillPresent.cleanedAt).not.toBeNull();
const afterCleaning = await findExpiredJobs(pool);
expect(afterCleaning).toHaveLength(0);
}); });
}); });
test/worker.test.js
+20 -15
View File
@@ -5,7 +5,7 @@ import os from 'node:os';
import { getPool, closePool } from '../src/db.js'; import { getPool, closePool } from '../src/db.js';
import { loadConfig } from '../src/config.js'; import { loadConfig } from '../src/config.js';
import { ensureStorageDirs, uploadPath, outputPath } from '../src/storage.js'; import { ensureStorageDirs, uploadPath, outputPath } from '../src/storage.js';
import { createJob, getJobById, getJobErrorLog } from '../src/jobs/jobRepository.js'; import { createJob, getJobByUuid, getJobErrorLog } from '../src/jobs/jobRepository.js';
import { registerImageConverters } from '../src/converters/image.js'; import { registerImageConverters } from '../src/converters/image.js';
import { processPendingJobs } from '../src/worker.js'; import { processPendingJobs } from '../src/worker.js';
@@ -28,61 +28,66 @@ beforeEach(async () => {
await pool.query('DELETE FROM conversion_jobs'); await pool.query('DELETE FROM conversion_jobs');
}); });
async function createPendingImageJob(id, sourceFormat, targetFormat) { async function createPendingImageJob(uuid, sourceFormat, targetFormat) {
const fixturePath = path.join(import.meta.dirname, 'fixtures', 'sample.png'); const fixturePath = path.join(import.meta.dirname, 'fixtures', 'sample.png');
const inputFilePath = uploadPath(config, id, sourceFormat); const inputFilePath = uploadPath(config, uuid, sourceFormat);
await fs.copyFile(fixturePath, inputFilePath); await fs.copyFile(fixturePath, inputFilePath);
const { size: inputSizeBytes } = await fs.stat(inputFilePath);
await createJob(pool, { await createJob(pool, {
id, uuid,
family: 'image', family: 'image',
sourceFormat, sourceFormat,
targetFormat, targetFormat,
originalFilename: `photo.${sourceFormat}`, originalFilename: `photo.${sourceFormat}`,
inputPath: `${id}.${sourceFormat}`, inputPath: `${uuid}.${sourceFormat}`,
inputMimeType: 'image/png', inputMimeType: 'image/png',
inputSizeBytes,
expiresAt: new Date(Date.now() + 3600 * 1000), expiresAt: new Date(Date.now() + 3600 * 1000),
}); });
} }
describe('processPendingJobs', () => { describe('processPendingJobs', () => {
it('converts a pending image job to done', async () => { it('converts a pending image job to done', async () => {
const id = '99999999-9999-4999-8999-999999999999'; const uuid = '99999999-9999-4999-8999-999999999999';
await createPendingImageJob(id, 'png', 'webp'); await createPendingImageJob(uuid, 'png', 'webp');
const processedCount = await processPendingJobs(pool, config); const processedCount = await processPendingJobs(pool, config);
expect(processedCount).toBe(1); expect(processedCount).toBe(1);
const job = await getJobById(pool, id); const job = await getJobByUuid(pool, uuid);
expect(job.status).toBe('done'); expect(job.status).toBe('done');
expect(job.outputPath).toBe(`${id}.webp`); expect(job.outputPath).toBe(`${uuid}.webp`);
expect(job.outputMimeType).toBe('image/webp'); expect(job.outputMimeType).toBe('image/webp');
expect(job.outputSizeBytes).toBeGreaterThan(0);
expect(Number(job.conversionDurationSeconds)).toBeGreaterThanOrEqual(0);
const stat = await fs.stat(outputPath(config, id, 'webp')); const stat = await fs.stat(outputPath(config, uuid, 'webp'));
expect(stat.size).toBeGreaterThan(0); expect(stat.size).toBeGreaterThan(0);
}); });
it('marks a job failed with a safe message and a detailed log when the converter throws', async () => { it('marks a job failed with a safe message and a detailed log when the converter throws', async () => {
const id = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; const uuid = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa';
await createJob(pool, { await createJob(pool, {
id, uuid,
family: 'image', family: 'image',
sourceFormat: 'png', sourceFormat: 'png',
targetFormat: 'webp', targetFormat: 'webp',
originalFilename: 'missing.png', originalFilename: 'missing.png',
inputPath: `${id}.png`, inputPath: `${uuid}.png`,
inputMimeType: 'image/png', inputMimeType: 'image/png',
inputSizeBytes: 11,
expiresAt: new Date(Date.now() + 3600 * 1000), expiresAt: new Date(Date.now() + 3600 * 1000),
}); });
// Note: input file is intentionally never written, so sharp will throw ENOENT. // Note: input file is intentionally never written, so sharp will throw ENOENT.
await processPendingJobs(pool, config); await processPendingJobs(pool, config);
const job = await getJobById(pool, id); const job = await getJobByUuid(pool, uuid);
expect(job.status).toBe('failed'); expect(job.status).toBe('failed');
expect(job.errorMessage).toBe('Conversion failed, please try again.'); expect(job.errorMessage).toBe('Conversion failed, please try again.');
const errorLog = await getJobErrorLog(pool, id); const errorLog = await getJobErrorLog(pool, job.id);
expect(errorLog).toMatch(/input file is missing/i); expect(errorLog).toMatch(/input file is missing/i);
}); });