Ombrora/convert
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add auto-increment id, size/duration tracking, and soft cleanup to conversion_jobs

Browse Source 
Rename the old CHAR(36) id to uuid (still used for public URLs and file
naming) and add a real auto-increment id as the primary key. Track
input/output file size and conversion duration per job. Cleanup no
longer deletes rows; it marks cleaned_at and skips already-cleaned
expired jobs on later runs.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
anthony
2026-07-30 22:06:32 +02:00
co-authored by Claude Sonnet 5
parent 24f7ab2625
commit ff5a257a0e
10 changed files with 208 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app.js
+9 -8
View File
@@ -10,7 +10,7 @@ import { registerImageToPdfConverter } from './converters/imageToPdf.js';
import { registerDocumentConverters } from './converters/document.js';
import { resolveInputFormat } from './mime.js';
import { deleteIfExists } from './storage.js';
import { createJob, getJobById } from './jobs/jobRepository.js';
import { createJob, getJobByUuid } from './jobs/jobRepository.js';
import { outputPath } from './storage.js';
let convertersRegistered = false;
@@ -77,7 +77,7 @@ export function createApp(config, pool) {
for (let i = 0; i < req.files.length; i += 1) {
const file = req.files[i];
const targetFormat = targetFormats[i];
const id = path.basename(file.filename, path.extname(file.filename));
const uuid = path.basename(file.filename, path.extname(file.filename));
const sourceFormat = path.extname(file.filename).slice(1).toLowerCase();
const { mime, valid } = await resolveInputFormat(file.path, sourceFormat);
@@ -99,30 +99,31 @@ export function createApp(config, pool) {
const expiresAt = new Date(Date.now() + config.retentionHours * 3600 * 1000);
await createJob(pool, {
id,
uuid,
family: registryEntry.family,
sourceFormat,
targetFormat,
originalFilename: file.originalname,
inputPath: file.filename,
inputMimeType: mime,
inputSizeBytes: file.size,
expiresAt,
});
results.push({ file: file.originalname, id, status: 'pending' });
results.push({ file: file.originalname, id: uuid, status: 'pending' });
}
res.status(201).json({ jobs: results });
});
app.get('/api/jobs/:id', async (req, res) => {
const job = await getJobById(pool, req.params.id);
const job = await getJobByUuid(pool, req.params.id);
if (!job) {
return res.status(404).json({ error: 'Job not found' });
}
res.json({
id: job.id,
id: job.uuid,
status: job.status,
originalFilename: job.originalFilename,
sourceFormat: job.sourceFormat,
@@ -138,7 +139,7 @@ export function createApp(config, pool) {
}
app.get('/api/jobs/:id/download', async (req, res) => {
const job = await getJobById(pool, req.params.id);
const job = await getJobByUuid(pool, req.params.id);
if (!job) {
return res.status(404).json({ error: 'Job not found' });
}
@@ -146,7 +147,7 @@ export function createApp(config, pool) {
return res.status(409).json({ error: `Job is not ready yet (status: ${job.status})` });
}
const filePath = outputPath(config, job.id, job.targetFormat);
const filePath = outputPath(config, job.uuid, job.targetFormat);
const downloadFilename = `${path.parse(job.originalFilename).name}.${job.targetFormat}`;
res.set('Content-Type', job.outputMimeType);
res.set('Content-Disposition', contentDispositionHeader(downloadFilename));
src/cleanup.js
+4 -4
View File
@@ -2,15 +2,15 @@ import { pathToFileURL } from 'node:url';
import { loadConfig } from './config.js';
import { getPool, closePool } from './db.js';
import { uploadPath, outputPath, deleteIfExists } from './storage.js';
import { findExpiredJobs, deleteJob } from './jobs/jobRepository.js';
import { findExpiredJobs, markCleaned } from './jobs/jobRepository.js';
export async function runCleanup(pool, config) {
const expiredJobs = await findExpiredJobs(pool);
for (const job of expiredJobs) {
await deleteIfExists(uploadPath(config, job.id, job.sourceFormat));
await deleteIfExists(outputPath(config, job.id, job.targetFormat));
await deleteJob(pool, job.id);
await deleteIfExists(uploadPath(config, job.uuid, job.sourceFormat));
await deleteIfExists(outputPath(config, job.uuid, job.targetFormat));
await markCleaned(pool, job.id);
}
return expiredJobs.length;
src/jobs/jobRepository.js
+29 -18
View File
@@ -2,6 +2,7 @@ function toCamelJob(row) {
if (!row) return null;
return {
id: row.id,
uuid: row.uuid,
status: row.status,
family: row.family,
sourceFormat: row.source_format,
@@ -11,38 +12,44 @@ function toCamelJob(row) {
outputPath: row.output_path,
inputMimeType: row.input_mime_type,
outputMimeType: row.output_mime_type,
inputSizeBytes: row.input_size_bytes,
outputSizeBytes: row.output_size_bytes,
conversionDurationSeconds: row.conversion_duration_seconds,
errorMessage: row.error_message,
createdAt: row.created_at,
updatedAt: row.updated_at,
expiresAt: row.expires_at,
cleanedAt: row.cleaned_at,
};
}
export async function createJob(pool, job) {
await pool.query(
`INSERT INTO conversion_jobs
(id, status, family, source_format, target_format, original_filename, input_path, input_mime_type, expires_at)
VALUES (?, 'pending', ?, ?, ?, ?, ?, ?, ?)`,
(uuid, status, family, source_format, target_format, original_filename, input_path, input_mime_type, input_size_bytes, expires_at)
VALUES (?, 'pending', ?, ?, ?, ?, ?, ?, ?, ?)`,
[
job.id,
job.uuid,
job.family,
job.sourceFormat,
job.targetFormat,
job.originalFilename,
job.inputPath,
job.inputMimeType,
job.inputSizeBytes,
job.expiresAt,
]
);
}
export async function getJobById(pool, id) {
export async function getJobByUuid(pool, uuid) {
const rows = await pool.query(
`SELECT id, status, family, source_format, target_format, original_filename,
`SELECT id, uuid, status, family, source_format, target_format, original_filename,
input_path, output_path, input_mime_type, output_mime_type,
error_message, created_at, updated_at, expires_at
FROM conversion_jobs WHERE id = ?`,
[id]
input_size_bytes, output_size_bytes, conversion_duration_seconds,
error_message, created_at, updated_at, expires_at, cleaned_at
FROM conversion_jobs WHERE uuid = ?`,
[uuid]
);
return toCamelJob(rows[0]);
}
@@ -56,10 +63,12 @@ export async function markProcessing(pool, id) {
await pool.query("UPDATE conversion_jobs SET status = 'processing' WHERE id = ?", [id]);
}
export async function markDone(pool, id, { outputPath, outputMimeType }) {
export async function markDone(pool, id, { outputPath, outputMimeType, outputSizeBytes, conversionDurationSeconds }) {
await pool.query(
"UPDATE conversion_jobs SET status = 'done', output_path = ?, output_mime_type = ? WHERE id = ?",
[outputPath, outputMimeType, id]
`UPDATE conversion_jobs
SET status = 'done', output_path = ?, output_mime_type = ?, output_size_bytes = ?, conversion_duration_seconds = ?
WHERE id = ?`,
[outputPath, outputMimeType, outputSizeBytes, conversionDurationSeconds, id]
);
}
@@ -72,9 +81,10 @@ export async function markFailed(pool, id, { errorMessage, errorLog }) {
export async function findPendingJobs(pool, limit) {
const rows = await pool.query(
`SELECT id, status, family, source_format, target_format, original_filename,
`SELECT id, uuid, status, family, source_format, target_format, original_filename,
input_path, output_path, input_mime_type, output_mime_type,
error_message, created_at, updated_at, expires_at
input_size_bytes, output_size_bytes, conversion_duration_seconds,
error_message, created_at, updated_at, expires_at, cleaned_at
FROM conversion_jobs WHERE status = 'pending' ORDER BY created_at ASC LIMIT ?`,
[limit]
);
@@ -83,14 +93,15 @@ export async function findPendingJobs(pool, limit) {
export async function findExpiredJobs(pool) {
const rows = await pool.query(
`SELECT id, status, family, source_format, target_format, original_filename,
`SELECT id, uuid, status, family, source_format, target_format, original_filename,
input_path, output_path, input_mime_type, output_mime_type,
error_message, created_at, updated_at, expires_at
FROM conversion_jobs WHERE expires_at < NOW()`
input_size_bytes, output_size_bytes, conversion_duration_seconds,
error_message, created_at, updated_at, expires_at, cleaned_at
FROM conversion_jobs WHERE expires_at < NOW() AND cleaned_at IS NULL`
);
return rows.map(toCamelJob);
}
export async function deleteJob(pool, id) {
await pool.query('DELETE FROM conversion_jobs WHERE id = ?', [id]);
export async function markCleaned(pool, id) {
await pool.query('UPDATE conversion_jobs SET cleaned_at = NOW() WHERE id = ?', [id]);
}
src/worker.js
+9 -3
View File
@@ -1,4 +1,5 @@
import { pathToFileURL } from 'node:url';
import fs from 'node:fs/promises';
import { loadConfig } from './config.js';
import { getPool } from './db.js';
import { ensureStorageDirs, uploadPath, outputPath, deleteIfExists } from './storage.js';
@@ -22,8 +23,8 @@ function withTimeout(promise, ms) {
async function processJob(pool, config, job) {
await markProcessing(pool, job.id);
const inputFilePath = uploadPath(config, job.id, job.sourceFormat);
const outputFilePath = outputPath(config, job.id, job.targetFormat);
const inputFilePath = uploadPath(config, job.uuid, job.sourceFormat);
const outputFilePath = outputPath(config, job.uuid, job.targetFormat);
try {
const entry = resolveConverter(job.sourceFormat, job.targetFormat);
@@ -31,11 +32,16 @@ async function processJob(pool, config, job) {
throw new Error(`No converter registered for ${job.sourceFormat} -> ${job.targetFormat}`);
}
const startedAt = Date.now();
await withTimeout(entry.convert(inputFilePath, outputFilePath), JOB_TIMEOUT_MS);
const conversionDurationSeconds = (Date.now() - startedAt) / 1000;
const { size: outputSizeBytes } = await fs.stat(outputFilePath);
await markDone(pool, job.id, {
outputPath: `${job.id}.${job.targetFormat}`,
outputPath: `${job.uuid}.${job.targetFormat}`,
outputMimeType: outputMimeType(job.targetFormat),
outputSizeBytes,
conversionDurationSeconds,
});
} catch (error) {
await deleteIfExists(outputFilePath);