"use strict"; var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", { value: true }); const express_1 = __importDefault(require("express")); const promises_1 = __importDefault(require("fs/promises")); const fs_1 = require("fs"); const os_1 = require("os"); const path_1 = require("path"); const crypto_1 = require("crypto"); const sqlite3_1 = require("sqlite3"); const body_parser_1 = __importDefault(require("body-parser")); const multer_1 = __importDefault(require("multer")); const uuid_1 = require("uuid"); const mime_1 = require("mime"); const port = typeof process.env['PORT'] !== 'undefined' ? parseInt(process.env['PORT'], 10) : 3333; const data = typeof process.env['DATADIR'] !== 'undefined' ? process.env['DATADIR'] : './data'; const dbPath = (0, path_1.join)(data, 'queue.sqlite'); const app = (0, express_1.default)(); const tmp = (0, os_1.tmpdir)(); const db = new sqlite3_1.Database(dbPath); const accepted = ['application/zip', 'application/x-zip-compressed']; const storage = multer_1.default.diskStorage({ destination: function (req, file, cb) { cb(null, tmp); }, filename: function (req, file, cb) { cb(null, `${+new Date()}_${file.originalname}`); } }); function fileFilter(req, file, cb) { if (accepted.indexOf(file.mimetype) !== -1) { cb(null, true); } else { console.warn(`Filetype ${file.mimetype} is not of type zip`); cb(new Error("Dataset is not of type zip"), false); } } const upload = (0, multer_1.default)({ storage, fileFilter }); app.use(body_parser_1.default.json()); app.use(body_parser_1.default.urlencoded({ extended: true })); function hash(path) { return new Promise((resolve, reject) => { const hashSum = (0, crypto_1.createHash)('sha256'); const stream = (0, fs_1.createReadStream)(path); stream.on('error', (err) => reject(err)); stream.on('data', (chunk) => hashSum.update(chunk)); stream.on('end', () => resolve(hashSum.digest('hex'))); }); } async function exists(path) { try { await promises_1.default.access(path); return true; } catch { return false; } } async function add(email, name, dataset, model) { const query = `INSERT INTO queue (id, email, name, dataset, model) VALUES ( ?, ?, ?, ?, ?);`; const id = (0, uuid_1.v4)(); return new Promise((resolve, reject) => { return db.run(query, [id, email, name, dataset, model], (err, row) => { if (err) return reject(err); console.log(`Added job ${id} to queue`); return resolve(id); }); }); } async function status(id) { const query = `SELECT name, model, started, completed, failed, meta FROM queue WHERE id = ? LIMIT 1;`; let jobStatus = 'Unknown'; return new Promise((resolve, reject) => { return db.all(query, [id], (err, rows) => { if (err) return reject(err); if (rows[0].started === null) { jobStatus = `Has not started`; } else if (rows[0].failed !== null) { jobStatus = `Failed
${rows[0].meta}
`; } else if (rows[0].completed !== null) { jobStatus = `Completed ${rows[0].completed} Download`; } console.log(`Got status for job ${id}: ${jobStatus}`); return resolve(jobStatus); }); }); } async function name(id) { const query = `SELECT name, meta FROM queue WHERE id = ? LIMIT 1;`; return new Promise((resolve, reject) => { return db.all(query, [id], (err, rows) => { if (err) return reject(err); if (rows.length < 1) { return reject(new Error(`Job ${id} does not exist`)); } return resolve(rows[0].name); }); }); } async function dataset(id) { const query = `SELECT dataset FROM queue WHERE id = ? LIMIT 1;`; return new Promise((resolve, reject) => { return db.all(query, [id], (err, rows) => { if (err) return reject(err); if (rows.length < 1) { return reject(new Error(`Dataset ${id} does not exist`)); } return resolve(rows[0].dataset); }); }); } async function job() { const query = `SELECT id FROM queue WHERE started IS NULL AND completed IS NULL AND failed IS NULL ORDER BY created ASC LIMIT 1;`; return new Promise((resolve, reject) => { return db.all(query, [], (err, rows) => { if (err) return reject(err); if (rows.length < 1) { return resolve(null); } return resolve(rows[0].id); }); }); } app.get('/', async (req, res, next) => { let html; try { html = await promises_1.default.readFile('./views/index.html', 'utf8'); } catch (err) { console.error(err); return next(err); } res.send(html); }); app.post('/', upload.single('dataset'), async (req, res, next) => { let fileHash; let filePath; let fileExists; let id; req.setTimeout(0); if (typeof req.file === 'undefined' && req.file === null) { console.error('No file in upload'); return next('ERROR: Please upload dataset as zip file'); } try { fileHash = await hash(req.file.path); } catch (err) { console.error(err); return next(`Error hashing file ${req.file.originalname}`); } filePath = (0, path_1.join)(data, `${fileHash}.zip`); try { fileExists = await exists(filePath); } catch (err) { console.error(err); } if (!fileExists) { try { await promises_1.default.rename(req.file.path, filePath); console.log(`Saved dataset with hash ${fileHash}`); } catch (err) { console.error(err); return next(err); } } else { console.warn(`Dataset with hash ${fileHash} already exists...`); try { await promises_1.default.unlink(req.file.path); } catch (err) { console.error(err); } } try { id = await add(req.body.email, req.body.name, fileHash, req.body.model); } catch (err) { console.log(err); return next(`Error adding training job ${req.body.name}`); } res.send(`Dataset for job ${req.body.name} has been uploaded successfully. You will be emailed when your job has started and when it has completed training.
Monitor job status here: ${id}`); }); app.get('/job/:id', async (req, res, next) => { let jobStatus; if (typeof req.params.id === 'undefined' || req.params.id === null) { console.error(`No job id provided`); return next('Invalid request'); } if (req.params.id.length !== 36) { console.error(`Job id ${req.params.id} is invalid`); return next('Invalid job id'); } try { jobStatus = await status(req.params.id); } catch (err) { console.error(err); return next('Error getting job status'); } return res.send(`Job: ${req.params.id}
Status: ${jobStatus}`); }); app.get('/model/:id', async (req, res, next) => { let filePath; let fileExists = false; let id; let fileName; let fileStream; let mimeType; let stream; if (typeof req.params.id === 'undefined' || req.params.id === null) { console.error(`No job id provided`); return next('Invalid request'); } id = req.params.id; filePath = (0, path_1.join)(data, `${id}.onnx`); try { fileExists = await exists(filePath); } catch (err) { console.error(err); return next(`Error checking whether model for job ${id} exists`); } if (!fileExists) { console.warn(`Model for job ${id} does not exist`); return next(`Model for job ${id} does not exist`); } try { fileName = await name(id); } catch (err) { console.error(err); return next(`Error getting job ${id}`); } mimeType = (0, mime_1.getType)(filePath); res.setHeader('Content-disposition', `attachment; filename=${fileName}.onnx`); res.setHeader('Content-type', mimeType); stream = (0, fs_1.createReadStream)(filePath); stream.pipe(res); }); app.get('/dataset/:id', async (req, res, next) => { let filePath; let fileExists = false; let id; let datasetHash; let fileStream; let mimeType; let stream; if (typeof req.params.id === 'undefined' || req.params.id === null) { console.error(`No dataset id provided`); return next('Invalid request'); } id = req.params.id; try { datasetHash = await dataset(id); } catch (err) { console.error(err); return next(`Error getting dataset for job ${id}`); } filePath = (0, path_1.join)(data, `${datasetHash}.zip`); try { fileExists = await exists(filePath); } catch (err) { console.error(err); return next(`Error checking whether dataset for job ${id} exists`); } if (!fileExists) { console.warn(`Dataset for job ${id} does not exist`); return next(`Dataset for job ${id} does not exist`); } mimeType = (0, mime_1.getType)(filePath); res.setHeader('Content-disposition', `attachment; filename=${datasetHash}.zip`); res.setHeader('Content-type', mimeType); stream = (0, fs_1.createReadStream)(filePath); stream.pipe(res); }); app.get('/job', async (req, res, next) => { let jobId; try { jobId = await job(); } catch (err) { console.error(err); return next('Error getting job'); } res.json([jobId]); }); //app.get('/jobs'); //app.post('/job/started/:id', ) app.listen(port, () => { console.log(`yolo_web running on port ${port}`); }); //# sourceMappingURL=index.js.map