yolo_web/dist/index.js

"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
    if (mod && mod.__esModule) return mod;
    var result = {};
    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
    __setModuleDefault(result, mod);
    return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
require("dotenv/config");
const express_1 = __importDefault(require("express"));
const promises_1 = __importDefault(require("fs/promises"));
const fs_1 = require("fs");
const os_1 = require("os");
const path_1 = require("path");
const crypto_1 = require("crypto");
const sqlite3_1 = require("sqlite3");
const body_parser_1 = __importDefault(require("body-parser"));
const multer_1 = __importDefault(require("multer"));
const uuid_1 = require("uuid");
const mime_1 = require("mime");
const log_1 = require("./log");
const mail_1 = require("./mail");
const Handlebars = __importStar(require("handlebars"));
const yoloWebUrl = typeof process.env['YOLO_WEB_URL'] !== 'undefined' ? process.env['YOLO_WEB_URL'] : 'http://localhost:3333';
const port = typeof process.env['PORT'] !== 'undefined' ? parseInt(process.env['PORT'], 10) : 3333;
const data = typeof process.env['DATADIR'] !== 'undefined' ? process.env['DATADIR'] : './data';
const dbPath = (0, path_1.join)(data, 'queue.sqlite');
const app = (0, express_1.default)();
const tmp = (0, os_1.tmpdir)();
const db = new sqlite3_1.Database(dbPath);
let log = (0, log_1.createLog)('server');
const accepted = [
    'application/octet-stream',
    'multipart/x-zip',
    'application/zip',
    'application/zip-compressed',
    'application/x-zip-compressed',
];
const storage = multer_1.default.diskStorage({
    destination: function (req, file, cb) {
        cb(null, tmp);
    },
    filename: function (req, file, cb) {
        cb(null, `${+new Date()}_${file.originalname}`);
    }
});
function fileFilter(req, file, cb) {
    if (accepted.indexOf(file.mimetype) !== -1) {
        cb(null, true);
    }
    else {
        log.warn(`Filetype "${file.mimetype}" is not of type zip`);
        cb(new Error("Dataset is not of type zip"), false);
    }
}
const uploadZip = (0, multer_1.default)({ storage, fileFilter });
const uploadOnnx = (0, multer_1.default)({ storage, fileFilter });
app.use(body_parser_1.default.json());
app.use(body_parser_1.default.urlencoded({ extended: true }));
async function createTemplate(filePath) {
    let tmpl;
    try {
        tmpl = await promises_1.default.readFile(filePath, 'utf8');
    }
    catch (err) {
        log.error(err);
        return null;
    }
    return Handlebars.compile(tmpl);
}
let index;
function hash(path) {
    return new Promise((resolve, reject) => {
        const hashSum = (0, crypto_1.createHash)('sha256');
        const stream = (0, fs_1.createReadStream)(path);
        stream.on('error', (err) => reject(err));
        stream.on('data', (chunk) => hashSum.update(chunk));
        stream.on('end', () => resolve(hashSum.digest('hex')));
    });
}
async function exists(path) {
    try {
        await promises_1.default.access(path);
        return true;
    }
    catch {
        return false;
    }
}
async function add(email, name, dataset, model) {
    const query = `INSERT INTO queue 
		(id, email, name, dataset, model) 
		VALUES ( ?, ?, ?, ?, ?);`;
    const id = (0, uuid_1.v4)();
    return new Promise((resolve, reject) => {
        return db.run(query, [id, email, name, dataset, model], (err, row) => {
            if (err)
                return reject(err);
            log.info(`Added job ${id} to queue`);
            return resolve(id);
        });
    });
}
async function status(id) {
    const query = `SELECT name, model, started, completed, failed, meta FROM queue WHERE id = ? LIMIT 1;`;
    let jobStatus = 'Unknown';
    return new Promise((resolve, reject) => {
        return db.all(query, [id], (err, rows) => {
            if (err)
                return reject(err);
            if (rows.length !== 1) {
                return resolve(jobStatus);
            }
            const obj = rows[0];
            if (rows[0].started === null) {
                jobStatus = `Has not started`;
            }
            else if (rows[0].failed !== null) {
                jobStatus = `Failed <br /> <pre>${rows[0].meta}</pre>`;
            }
            else if (rows[0].completed !== null) {
                jobStatus = `Completed ${rows[0].completed} <a href="/model/${id}">Download</a>`;
            }
            else if (rows[0].started !== null) {
                jobStatus = `Started ${rows[0].started}`;
            }
            log.info(`Got status for job ${id}: ${jobStatus}`);
            return resolve(jobStatus);
        });
    });
}
async function name(id) {
    const query = `SELECT name, meta FROM queue WHERE id = ? LIMIT 1;`;
    return new Promise((resolve, reject) => {
        return db.all(query, [id], (err, rows) => {
            if (err)
                return reject(err);
            if (rows.length < 1) {
                return reject(new Error(`Job ${id} does not exist`));
            }
            return resolve(rows[0].name);
        });
    });
}
async function dataset(id) {
    const query = `SELECT dataset FROM queue WHERE id = ? LIMIT 1;`;
    return new Promise((resolve, reject) => {
        return db.all(query, [id], (err, rows) => {
            if (err)
                return reject(err);
            if (rows.length < 1) {
                return reject(new Error(`Dataset ${id} does not exist`));
            }
            return resolve(rows[0].dataset);
        });
    });
}
async function job() {
    const query = `SELECT id FROM queue WHERE 
			started IS NULL 
		AND completed IS NULL 
		AND failed IS NULL
		ORDER BY created ASC
		LIMIT 1;`;
    return new Promise((resolve, reject) => {
        return db.all(query, [], (err, rows) => {
            if (err)
                return reject(err);
            if (rows.length < 1) {
                return resolve([]);
            }
            return resolve([rows[0].id]);
        });
    });
}
async function jobs() {
    const query = `SELECT id FROM queue WHERE 
			started IS NULL 
		AND completed IS NULL 
		AND failed IS NULL
		ORDER BY created ASC;`;
    return new Promise((resolve, reject) => {
        return db.all(query, [], (err, rows) => {
            if (err)
                return reject(err);
            return resolve(rows.map((el) => el.id));
        });
    });
}
async function claim(id) {
    const query = `SELECT * FROM queue WHERE id = ? LIMIT 1;`;
    return new Promise((resolve, reject) => {
        return db.all(query, [id], (err, rows) => {
            if (err)
                return reject(err);
            if (rows.length < 1) {
                return reject(new Error(`Dataset ${id} does not exist`));
            }
            if (rows[0].started !== null) {
                return reject(new Error(`Job ${id} is already claimed`));
            }
            const claimQuery = `UPDATE queue SET started = CURRENT_TIMESTAMP WHERE id = ?;`;
            return db.run(claimQuery, [id], (err, row) => {
                if (err)
                    return reject(err);
                return resolve(rows[0]);
            });
        });
    });
}
async function get(id) {
    const query = `SELECT * FROM queue WHERE id = ? LIMIT 1;`;
    return new Promise((resolve, reject) => {
        return db.all(query, [id], (err, rows) => {
            if (err)
                return reject(err);
            if (rows.length < 1) {
                return reject(new Error(`Job ${id} does not exist`));
            }
            return resolve(rows[0]);
        });
    });
}
async function fail(id, meta) {
    const query = `UPDATE queue SET failed = CURRENT_TIMESTAMP, meta = ? WHERE id = ?;`;
    return new Promise((resolve, reject) => {
        return db.run(query, [meta, id], (err, row) => {
            if (err)
                return reject(err);
            return resolve(true);
        });
    });
}
async function complete(id, meta) {
    const query = `UPDATE queue SET completed = CURRENT_TIMESTAMP, meta = ? WHERE id = ?;`;
    return new Promise((resolve, reject) => {
        return db.run(query, [meta, id], (err, row) => {
            if (err)
                return reject(err);
            return resolve(true);
        });
    });
}
async function all() {
    const query = `SELECT * FROM queue ORDER BY created DESC;`;
    return new Promise((resolve, reject) => {
        return db.all(query, [], (err, rows) => {
            if (err)
                return reject(err);
            return resolve(rows);
        });
    });
}
function annotate(row) {
    row.hasModel = row.completed != null;
    if (row.completed != null) {
        row.status = 'Completed';
    }
    else if (row.failed != null) {
        row.status = 'Failed';
    }
    else if (row.started != null) {
        row.status = 'Training';
    }
    else {
        row.status = 'Waiting';
    }
    return row;
}
async function alertClaimed(id, model, name, email) {
    const subject = `Training ${name} started`;
    const body = `<div><h3>Your ${model} training job "${name}" has started!</h3>
<br />
<div>Status is available here: <a href="${yoloWebUrl}/job/${id}">${yoloWebUrl}/job/${id}</a><div>
<br />
<div>You will receive an email when the training is complete.</div>
</div>`;
    try {
        await (0, mail_1.sendMail)(email, subject, body);
    }
    catch (err) {
        log.error('Error sending mail');
        log.error(err);
    }
}
async function alertFailed(id, model, name, email) {
    const subject = `Training ${name} failed`;
    const body = `<div><h3>Your ${model} training job "${name}" has failed :(</h3>
<br />
<div>Additional information is available here: <a href="${yoloWebUrl}/job/${id}">${yoloWebUrl}/job/${id}</a><div>
<br />
<div>Please contact the administrator for more information</div>
</div>`;
    try {
        await (0, mail_1.sendMail)(email, subject, body);
    }
    catch (err) {
        log.error('Error sending mail');
        log.error(err);
    }
}
async function alertCompleted(id, model, name, email) {
    const subject = `Training ${name} completed`;
    const body = `<div><h3>Your ${model} training job "${name}" has completed!</h3>
<br />
<div>The model is available for download here: <a href="${yoloWebUrl}/model/${id}">${yoloWebUrl}/model/${id}</a><div>
</div>`;
    try {
        await (0, mail_1.sendMail)(email, subject, body);
    }
    catch (err) {
        log.error('Error sending mail');
        log.error(err);
    }
}
app.get('/', async (req, res, next) => {
    let html;
    let rows;
    let input;
    try {
        rows = await all();
    }
    catch (err) {
        log.error(err);
        return next('ERROR: Could not retrieve jobs from queue');
    }
    rows = rows.map(annotate);
    input = {
        rows,
        hasTable: typeof rows == 'undefined' ? false : rows.length > 0
    };
    html = index(input);
    res.send(html);
});
app.post('/', uploadZip.single('dataset'), async (req, res, next) => {
    let fileHash;
    let filePath;
    let fileExists;
    let id;
    req.setTimeout(0);
    if (typeof req.file === 'undefined' || req.file === null) {
        log.error('No file in upload');
        return next('ERROR: Please upload dataset as zip file');
    }
    try {
        fileHash = await hash(req.file.path);
    }
    catch (err) {
        log.error(err);
        return next(`Error hashing file ${req.file.originalname}`);
    }
    filePath = (0, path_1.join)(data, `${fileHash}.zip`);
    try {
        fileExists = await exists(filePath);
    }
    catch (err) {
        log.error(err);
    }
    if (!fileExists) {
        try {
            await promises_1.default.copyFile(req.file.path, filePath);
            log.info(`Saved dataset with hash ${fileHash}`);
        }
        catch (err) {
            log.error(err);
            return next(err);
        }
    }
    else {
        log.warn(`Dataset with hash ${fileHash} already exists...`);
    }
    try {
        await promises_1.default.unlink(req.file.path);
    }
    catch (err) {
        log.error(err);
    }
    try {
        id = await add(req.body.email, req.body.name, fileHash, req.body.model);
    }
    catch (err) {
        log.info(err);
        return next(`Error adding training job ${req.body.name}`);
    }
    res.send(`<html><body>Dataset for job ${req.body.name} has been uploaded successfully. You will be emailed when your job has started and when it has completed training. <br /> Monitor job status here: <a href="/job/${id}">${id}</a></body></html>`);
});
app.post('/job/:id', uploadOnnx.single('model'), async (req, res, next) => {
    let filePath;
    let meta = null;
    let id;
    let jobObj;
    req.setTimeout(0);
    if (typeof req.file === 'undefined' || req.file === null) {
        log.error('No file in upload');
        return next('ERROR: Please model as zip file');
    }
    id = req.params.id;
    filePath = (0, path_1.join)(data, `${id}.zip`);
    if (typeof req.body.meta !== 'undefined') {
        meta = req.body.meta;
    }
    try {
        await promises_1.default.copyFile(req.file.path, filePath);
        log.info(`Saved model for job ${id}`);
    }
    catch (err) {
        log.error(err);
        return next(err);
    }
    try {
        await promises_1.default.unlink(req.file.path);
    }
    catch (err) {
        log.error(err);
    }
    try {
        await complete(id, meta);
    }
    catch (err) {
        log.error(err);
        return next(`Error completing training job ${id}`);
    }
    try {
        jobObj = await get(id);
    }
    catch (err) {
        log.error('Error getting job for alertCompleted');
        log.error(err);
    }
    try {
        await alertCompleted(id, jobObj.model, jobObj.name, jobObj.email);
    }
    catch (err) {
        log.error('Error sending alertCompleted email');
    }
    res.json({ id });
});
app.get('/job/:id', async (req, res, next) => {
    let jobStatus;
    if (typeof req.params.id === 'undefined' || req.params.id === null) {
        log.error(`No job id provided`);
        return next('Invalid request');
    }
    if (req.params.id.length !== 36) {
        log.error(`Job id ${req.params.id} is invalid`);
        return next('Invalid job id');
    }
    try {
        jobStatus = await status(req.params.id);
    }
    catch (err) {
        log.error(err);
        return next('Error getting job status');
    }
    return res.send(`<html><body>Job: ${req.params.id}<br /> Status: ${jobStatus}</body></html>`);
});
app.get('/model/:id', async (req, res, next) => {
    let filePath;
    let fileExists = false;
    let id;
    let fileName;
    let fileStream;
    let mimeType;
    let stream;
    if (typeof req.params.id === 'undefined' || req.params.id === null) {
        log.error(`No job id provided`);
        return next('Invalid request');
    }
    id = req.params.id;
    filePath = (0, path_1.join)(data, `${id}.zip`);
    try {
        fileExists = await exists(filePath);
    }
    catch (err) {
        log.error(err);
        return next(`Error checking whether model for job ${id} exists`);
    }
    if (!fileExists) {
        log.warn(`Model for job ${id} does not exist`);
        return next(`Model for job ${id} does not exist`);
    }
    try {
        fileName = await name(id);
    }
    catch (err) {
        log.error(err);
        return next(`Error getting job ${id}`);
    }
    mimeType = (0, mime_1.getType)(filePath);
    res.setHeader('Content-disposition', `attachment; filename=${fileName}.zip`);
    res.setHeader('Content-type', mimeType);
    stream = (0, fs_1.createReadStream)(filePath);
    stream.pipe(res);
});
app.get('/dataset/:id', async (req, res, next) => {
    let filePath;
    let fileExists = false;
    let id;
    let datasetHash;
    let fileStream;
    let mimeType;
    let stream;
    if (typeof req.params.id === 'undefined' || req.params.id === null) {
        log.error(`No dataset id provided`);
        return next('Invalid request');
    }
    id = req.params.id;
    try {
        datasetHash = await dataset(id);
    }
    catch (err) {
        log.error(err);
        return next(`Error getting dataset for job ${id}`);
    }
    filePath = (0, path_1.join)(data, `${datasetHash}.zip`);
    try {
        fileExists = await exists(filePath);
    }
    catch (err) {
        log.error(err);
        return next(`Error checking whether dataset for job ${id} exists`);
    }
    if (!fileExists) {
        log.warn(`Dataset for job ${id} does not exist`);
        return next(`Dataset for job ${id} does not exist`);
    }
    mimeType = (0, mime_1.getType)(filePath);
    res.setHeader('Content-disposition', `attachment; filename=${datasetHash}.zip`);
    res.setHeader('Content-type', mimeType);
    stream = (0, fs_1.createReadStream)(filePath);
    stream.pipe(res);
});
app.get('/job', async (req, res, next) => {
    let jobArr;
    try {
        jobArr = await job();
    }
    catch (err) {
        log.error(err);
        return next('Error getting job');
    }
    res.json(jobArr);
});
app.get('/jobs', async (req, res, next) => {
    let jobArr;
    try {
        jobArr = await jobs();
    }
    catch (err) {
        log.error(err);
        return next('Error getting job');
    }
    res.json(jobArr);
});
app.post('/job/claim/:id', async (req, res, next) => {
    let id;
    let jobObj;
    let resObj = {};
    if (typeof req.params.id === 'undefined' || req.params.id === null) {
        log.error(`No dataset id provided`);
        return next('Invalid request');
    }
    id = req.params.id;
    try {
        jobObj = await claim(id);
        log.info(`Job ${id} was claimed`);
    }
    catch (err) {
        log.error(err);
        return next('Error claiming job');
    }
    try {
        await alertClaimed(id, jobObj.model, jobObj.name, jobObj.email);
    }
    catch (err) {
        log.error(err);
    }
    resObj.id = id;
    resObj.path = `/dataset/${id}`;
    resObj.dataset = jobObj.dataset;
    resObj.model = jobObj.model;
    resObj.name = jobObj.name;
    resObj.email = jobObj.email;
    res.json(resObj);
});
app.post('/job/fail/:id', async (req, res, next) => {
    let id;
    let meta = null;
    let jobObj;
    if (typeof req.params.id === 'undefined' || req.params.id === null) {
        log.error(`No dataset id provided`);
        return next('Invalid request');
    }
    id = req.params.id;
    if (typeof req.body.meta !== 'undefined') {
        meta = req.body.meta;
    }
    try {
        await fail(id, meta);
        log.info(`Job ${id} failed`);
    }
    catch (err) {
        log.error(err);
        return next('Error failing job');
    }
    try {
        jobObj = await get(id);
    }
    catch (err) {
        log.error('Error getting job for alertFailed');
        log.error(err);
    }
    try {
        await alertFailed(id, jobObj.model, jobObj.name, jobObj.email);
    }
    catch (err) {
        log.error('Error sending alertFailed email');
    }
    res.json(true);
});
app.listen(port, async () => {
    index = await createTemplate('./views/index.hbs');
    log.info(`yolo_web running on port ${port}`);
});
//# sourceMappingURL=index.js.map