feat(archive): wire archive converters, validation, and extension parsing into app.js/worker.js
Also fixes a third double-extension bug the plan missed: app.js derived each job's uuid via path.basename(file.filename, path.extname(...)), which only strips the last dot segment. For a compound extension like tar.gz this left "<uuid>.tar" as the "uuid", overflowing the uuid column (Char(36)) and crashing the request. Replaced with stripExtension(). Also bumped rateLimitMaxJobs in jobs.test.js's config override — the file's cumulative POST /api/jobs calls across all tests now exceeds the production default (20/window) once the archive tests are included, causing spurious connection resets independent of any real bug. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+9
-5
@@ -13,6 +13,8 @@ import { registerHeicConverter } from './converters/heic.js';
|
|||||||
import { registerFontConverter } from './converters/font.js';
|
import { registerFontConverter } from './converters/font.js';
|
||||||
import { registerDfontConverter } from './converters/dfont.js';
|
import { registerDfontConverter } from './converters/dfont.js';
|
||||||
import { registerEbookConverter } from './converters/ebook.js';
|
import { registerEbookConverter } from './converters/ebook.js';
|
||||||
|
import { registerArchiveConverters } from './converters/archive.js';
|
||||||
|
import { extractExtension, stripExtension } from './archiveExtensions.js';
|
||||||
import { resolveInputFormat } from './mime.js';
|
import { resolveInputFormat } from './mime.js';
|
||||||
import { deleteIfExists } from './storage.js';
|
import { deleteIfExists } from './storage.js';
|
||||||
import { createJob, getJobByUuid } from './jobs/jobRepository.js';
|
import { createJob, getJobByUuid } from './jobs/jobRepository.js';
|
||||||
@@ -23,8 +25,9 @@ const VALID_ICON_SIZES = [16, 32, 48, 256, 512];
|
|||||||
function isValidQuality(targetFormat, quality) {
|
function isValidQuality(targetFormat, quality) {
|
||||||
if (quality === null || quality === undefined) return true;
|
if (quality === null || quality === undefined) return true;
|
||||||
if (!Number.isInteger(quality)) return false;
|
if (!Number.isInteger(quality)) return false;
|
||||||
if (targetFormat === 'gif' || targetFormat === 'ico') return false;
|
if (targetFormat === 'gif' || targetFormat === 'ico' || targetFormat === 'tar') return false;
|
||||||
if (targetFormat === 'png') return quality >= 0 && quality <= 9;
|
if (targetFormat === 'png') return quality >= 0 && quality <= 9;
|
||||||
|
if (['zip', 'tar.gz', 'tar.bz2', '7z', 'tar.7z'].includes(targetFormat)) return quality >= 0 && quality <= 9;
|
||||||
return quality >= 1 && quality <= 100;
|
return quality >= 1 && quality <= 100;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -46,6 +49,7 @@ function registerAllConverters() {
|
|||||||
registerFontConverter();
|
registerFontConverter();
|
||||||
registerDfontConverter();
|
registerDfontConverter();
|
||||||
registerEbookConverter();
|
registerEbookConverter();
|
||||||
|
registerArchiveConverters();
|
||||||
convertersRegistered = true;
|
convertersRegistered = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -58,7 +62,7 @@ export function createApp(config, prisma) {
|
|||||||
const storage = multer.diskStorage({
|
const storage = multer.diskStorage({
|
||||||
destination: (req, file, cb) => cb(null, path.join(config.storageDir, 'uploads')),
|
destination: (req, file, cb) => cb(null, path.join(config.storageDir, 'uploads')),
|
||||||
filename: (req, file, cb) => {
|
filename: (req, file, cb) => {
|
||||||
const ext = path.extname(file.originalname).slice(1).toLowerCase();
|
const ext = extractExtension(file.originalname);
|
||||||
cb(null, `${uuidv4()}.${ext}`);
|
cb(null, `${uuidv4()}.${ext}`);
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -137,8 +141,8 @@ export function createApp(config, prisma) {
|
|||||||
for (let i = 0; i < req.files.length; i += 1) {
|
for (let i = 0; i < req.files.length; i += 1) {
|
||||||
const file = req.files[i];
|
const file = req.files[i];
|
||||||
const targetFormat = targetFormats[i];
|
const targetFormat = targetFormats[i];
|
||||||
const uuid = path.basename(file.filename, path.extname(file.filename));
|
const uuid = stripExtension(file.filename);
|
||||||
const sourceFormat = path.extname(file.filename).slice(1).toLowerCase();
|
const sourceFormat = extractExtension(file.filename);
|
||||||
|
|
||||||
const { mime, valid } = await resolveInputFormat(file.path, sourceFormat);
|
const { mime, valid } = await resolveInputFormat(file.path, sourceFormat);
|
||||||
if (!valid) {
|
if (!valid) {
|
||||||
@@ -231,7 +235,7 @@ export function createApp(config, prisma) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const filePath = outputPath(config, job.uuid, job.targetFormat);
|
const filePath = outputPath(config, job.uuid, job.targetFormat);
|
||||||
const downloadFilename = `${path.parse(job.originalFilename).name}.${job.targetFormat}`;
|
const downloadFilename = `${stripExtension(job.originalFilename)}.${job.targetFormat}`;
|
||||||
res.set('Content-Type', job.outputMimeType);
|
res.set('Content-Type', job.outputMimeType);
|
||||||
res.set('Content-Disposition', contentDispositionHeader(downloadFilename));
|
res.set('Content-Disposition', contentDispositionHeader(downloadFilename));
|
||||||
fs.createReadStream(filePath).pipe(res);
|
fs.createReadStream(filePath).pipe(res);
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import { registerHeicConverter } from './converters/heic.js';
|
|||||||
import { registerFontConverter } from './converters/font.js';
|
import { registerFontConverter } from './converters/font.js';
|
||||||
import { registerDfontConverter } from './converters/dfont.js';
|
import { registerDfontConverter } from './converters/dfont.js';
|
||||||
import { registerEbookConverter } from './converters/ebook.js';
|
import { registerEbookConverter } from './converters/ebook.js';
|
||||||
|
import { registerArchiveConverters } from './converters/archive.js';
|
||||||
import { findPendingJobs, markProcessing, markDone, markFailed } from './jobs/jobRepository.js';
|
import { findPendingJobs, markProcessing, markDone, markFailed } from './jobs/jobRepository.js';
|
||||||
|
|
||||||
const JOB_TIMEOUT_MS = 60000;
|
const JOB_TIMEOUT_MS = 60000;
|
||||||
@@ -89,6 +90,7 @@ async function main() {
|
|||||||
registerFontConverter();
|
registerFontConverter();
|
||||||
registerDfontConverter();
|
registerDfontConverter();
|
||||||
registerEbookConverter();
|
registerEbookConverter();
|
||||||
|
registerArchiveConverters();
|
||||||
|
|
||||||
startWorker(prisma, config);
|
startWorker(prisma, config);
|
||||||
console.log(`Worker started, polling every ${config.workerPollIntervalMs}ms`);
|
console.log(`Worker started, polling every ${config.workerPollIntervalMs}ms`);
|
||||||
|
|||||||
+103
-1
@@ -8,13 +8,20 @@ import { getPrismaClient, closePrismaClient } from '../../src/db.js';
|
|||||||
import { loadConfig } from '../../src/config.js';
|
import { loadConfig } from '../../src/config.js';
|
||||||
import { getJobByUuid } from '../../src/jobs/jobRepository.js';
|
import { getJobByUuid } from '../../src/jobs/jobRepository.js';
|
||||||
import { ensureStorageDirs } from '../../src/storage.js';
|
import { ensureStorageDirs } from '../../src/storage.js';
|
||||||
|
import AdmZip from 'adm-zip';
|
||||||
|
|
||||||
let app;
|
let app;
|
||||||
let prisma;
|
let prisma;
|
||||||
let config;
|
let config;
|
||||||
|
|
||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
config = { ...loadConfig(), storageDir: await fs.mkdtemp(path.join(os.tmpdir(), 'converter-api-')) };
|
config = {
|
||||||
|
...loadConfig(),
|
||||||
|
storageDir: await fs.mkdtemp(path.join(os.tmpdir(), 'converter-api-')),
|
||||||
|
// This file's cumulative POST /api/jobs calls across all tests exceeds the
|
||||||
|
// production-tuned default (20/window) once the archive tests are included.
|
||||||
|
rateLimitMaxJobs: 1000,
|
||||||
|
};
|
||||||
await ensureStorageDirs(config);
|
await ensureStorageDirs(config);
|
||||||
prisma = getPrismaClient(config);
|
prisma = getPrismaClient(config);
|
||||||
app = createApp(config, prisma);
|
app = createApp(config, prisma);
|
||||||
@@ -343,3 +350,98 @@ describe('POST /api/jobs', () => {
|
|||||||
expect(response.status).toBe(400);
|
expect(response.status).toBe(400);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
function buildZipFixture(destPath) {
|
||||||
|
const zip = new AdmZip();
|
||||||
|
zip.addFile('hello.txt', Buffer.from('hello world'));
|
||||||
|
zip.writeZip(destPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('POST /api/jobs — archives', () => {
|
||||||
|
it('creates a pending job converting a zip upload to tar.gz', async () => {
|
||||||
|
const fixturePath = path.join(config.storageDir, 'archive.zip');
|
||||||
|
buildZipFixture(fixturePath);
|
||||||
|
|
||||||
|
const response = await request(app)
|
||||||
|
.post('/api/jobs')
|
||||||
|
.field('targetFormats', JSON.stringify(['tar.gz']))
|
||||||
|
.attach('files', fixturePath, 'archive.zip');
|
||||||
|
|
||||||
|
expect(response.status).toBe(201);
|
||||||
|
expect(response.body.jobs[0].status).toBe('pending');
|
||||||
|
|
||||||
|
const job = await getJobByUuid(prisma, response.body.jobs[0].id);
|
||||||
|
expect(job.sourceFormat).toBe('zip');
|
||||||
|
expect(job.targetFormat).toBe('tar.gz');
|
||||||
|
expect(job.family).toBe('archive');
|
||||||
|
|
||||||
|
await fs.unlink(fixturePath);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('creates a pending job with a compression level for a same-format zip -> zip conversion', async () => {
|
||||||
|
const fixturePath = path.join(config.storageDir, 'recompress.zip');
|
||||||
|
buildZipFixture(fixturePath);
|
||||||
|
|
||||||
|
const response = await request(app)
|
||||||
|
.post('/api/jobs')
|
||||||
|
.field('targetFormats', JSON.stringify(['zip']))
|
||||||
|
.field('qualities', JSON.stringify([9]))
|
||||||
|
.attach('files', fixturePath, 'recompress.zip');
|
||||||
|
|
||||||
|
expect(response.status).toBe(201);
|
||||||
|
const job = await getJobByUuid(prisma, response.body.jobs[0].id);
|
||||||
|
expect(job.targetFormat).toBe('zip');
|
||||||
|
expect(job.quality).toBe(9);
|
||||||
|
|
||||||
|
await fs.unlink(fixturePath);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects an out-of-range compression level for a zip target', async () => {
|
||||||
|
const fixturePath = path.join(config.storageDir, 'bad-level.zip');
|
||||||
|
buildZipFixture(fixturePath);
|
||||||
|
|
||||||
|
const response = await request(app)
|
||||||
|
.post('/api/jobs')
|
||||||
|
.field('targetFormats', JSON.stringify(['zip']))
|
||||||
|
.field('qualities', JSON.stringify([15]))
|
||||||
|
.attach('files', fixturePath, 'bad-level.zip');
|
||||||
|
|
||||||
|
expect(response.status).toBe(201);
|
||||||
|
expect(response.body.jobs[0].error).toMatch(/Invalid quality/);
|
||||||
|
|
||||||
|
await fs.unlink(fixturePath);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('rejects a compression level for a tar target (no compression to control)', async () => {
|
||||||
|
const fixturePath = path.join(config.storageDir, 'for-tar.zip');
|
||||||
|
buildZipFixture(fixturePath);
|
||||||
|
|
||||||
|
const response = await request(app)
|
||||||
|
.post('/api/jobs')
|
||||||
|
.field('targetFormats', JSON.stringify(['tar']))
|
||||||
|
.field('qualities', JSON.stringify([5]))
|
||||||
|
.attach('files', fixturePath, 'for-tar.zip');
|
||||||
|
|
||||||
|
expect(response.status).toBe(201);
|
||||||
|
expect(response.body.jobs[0].error).toMatch(/Invalid quality/);
|
||||||
|
|
||||||
|
await fs.unlink(fixturePath);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('parses a compound .tar.gz original filename as a single source format, not "gz"', async () => {
|
||||||
|
const zlib = await import('node:zlib');
|
||||||
|
const fixturePath = path.join(config.storageDir, 'backup.tar.gz');
|
||||||
|
await fs.writeFile(fixturePath, zlib.gzipSync(Buffer.from('irrelevant payload for this check')));
|
||||||
|
|
||||||
|
const response = await request(app)
|
||||||
|
.post('/api/jobs')
|
||||||
|
.field('targetFormats', JSON.stringify(['zip']))
|
||||||
|
.attach('files', fixturePath, 'backup.tar.gz');
|
||||||
|
|
||||||
|
expect(response.status).toBe(201);
|
||||||
|
const job = await getJobByUuid(prisma, response.body.jobs[0].id);
|
||||||
|
expect(job.sourceFormat).toBe('tar.gz');
|
||||||
|
|
||||||
|
await fs.unlink(fixturePath);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user