Compare commits
No commits in common. "82a7d598fea2cf419893b39cb8d3a0e128d75a0d" and "eea288b28d2ea31bb37b3835fac8f5059571e36c" have entirely different histories.
82a7d598fe
...
eea288b28d
|
@ -1,3 +1,2 @@
|
||||||
node_modules
|
node_modules
|
||||||
data/*
|
data/*
|
||||||
.env
|
|
|
@ -2,9 +2,8 @@ FROM node:lts-alpine
|
||||||
|
|
||||||
WORKDIR /code
|
WORKDIR /code
|
||||||
|
|
||||||
|
COPY ./dist /code/dist
|
||||||
COPY ./package*.json /code/
|
COPY ./package*.json /code/
|
||||||
RUN npm install --only-production
|
RUN npm install --only-production
|
||||||
|
|
||||||
COPY ./dist /code/dist
|
|
||||||
|
|
||||||
CMD ["npm", "run", "start"]
|
CMD ["npm", "run", "start"]
|
|
@ -1,2 +0,0 @@
|
||||||
YOLO_WEB_URL=http://localhost:3333
|
|
||||||
YOLOv5=../yolo_train/
|
|
|
@ -38,8 +38,7 @@ function fileFilter(req, file, cb) {
|
||||||
cb(new Error("Dataset is not of type zip"), false);
|
cb(new Error("Dataset is not of type zip"), false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const uploadZip = (0, multer_1.default)({ storage, fileFilter });
|
const upload = (0, multer_1.default)({ storage, fileFilter });
|
||||||
const uploadOnnx = (0, multer_1.default)({ storage });
|
|
||||||
app.use(body_parser_1.default.json());
|
app.use(body_parser_1.default.json());
|
||||||
app.use(body_parser_1.default.urlencoded({ extended: true }));
|
app.use(body_parser_1.default.urlencoded({ extended: true }));
|
||||||
function hash(path) {
|
function hash(path) {
|
||||||
|
@ -81,10 +80,6 @@ async function status(id) {
|
||||||
return db.all(query, [id], (err, rows) => {
|
return db.all(query, [id], (err, rows) => {
|
||||||
if (err)
|
if (err)
|
||||||
return reject(err);
|
return reject(err);
|
||||||
if (rows.length !== 1) {
|
|
||||||
return resolve(jobStatus);
|
|
||||||
}
|
|
||||||
const obj = rows[0];
|
|
||||||
if (rows[0].started === null) {
|
if (rows[0].started === null) {
|
||||||
jobStatus = `Has not started`;
|
jobStatus = `Has not started`;
|
||||||
}
|
}
|
||||||
|
@ -94,9 +89,6 @@ async function status(id) {
|
||||||
else if (rows[0].completed !== null) {
|
else if (rows[0].completed !== null) {
|
||||||
jobStatus = `Completed ${rows[0].completed} <a href="/model/${rows[0].model}/${id}">Download</a>`;
|
jobStatus = `Completed ${rows[0].completed} <a href="/model/${rows[0].model}/${id}">Download</a>`;
|
||||||
}
|
}
|
||||||
else if (rows[0].started !== null) {
|
|
||||||
jobStatus = `Started ${rows[0].started}`;
|
|
||||||
}
|
|
||||||
console.log(`Got status for job ${id}: ${jobStatus}`);
|
console.log(`Got status for job ${id}: ${jobStatus}`);
|
||||||
return resolve(jobStatus);
|
return resolve(jobStatus);
|
||||||
});
|
});
|
||||||
|
@ -140,64 +132,9 @@ async function job() {
|
||||||
if (err)
|
if (err)
|
||||||
return reject(err);
|
return reject(err);
|
||||||
if (rows.length < 1) {
|
if (rows.length < 1) {
|
||||||
return resolve([]);
|
return resolve(null);
|
||||||
}
|
}
|
||||||
return resolve([rows[0].id]);
|
return resolve(rows[0].id);
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
async function jobs() {
|
|
||||||
const query = `SELECT id FROM queue WHERE
|
|
||||||
started IS NULL
|
|
||||||
AND completed IS NULL
|
|
||||||
AND failed IS NULL
|
|
||||||
ORDER BY created ASC;`;
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
return db.all(query, [], (err, rows) => {
|
|
||||||
if (err)
|
|
||||||
return reject(err);
|
|
||||||
return resolve(rows.map((el) => el.id));
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
async function claim(id) {
|
|
||||||
const query = `SELECT * FROM queue WHERE id = ? LIMIT 1;`;
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
return db.all(query, [id], (err, rows) => {
|
|
||||||
if (err)
|
|
||||||
return reject(err);
|
|
||||||
if (rows.length < 1) {
|
|
||||||
return reject(new Error(`Dataset ${id} does not exist`));
|
|
||||||
}
|
|
||||||
if (rows[0].started !== null) {
|
|
||||||
return reject(new Error(`Job ${id} is already claimed`));
|
|
||||||
}
|
|
||||||
const claimQuery = `UPDATE queue SET started = CURRENT_TIMESTAMP WHERE id = ?;`;
|
|
||||||
return db.run(claimQuery, [id], (err, row) => {
|
|
||||||
if (err)
|
|
||||||
return reject(err);
|
|
||||||
return resolve(rows[0]);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
async function fail(id, meta) {
|
|
||||||
const query = `UPDATE queue SET failed = CURRENT_TIMESTAMP, meta = ? WHERE id = ?;`;
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
return db.run(query, [meta, id], (err, row) => {
|
|
||||||
if (err)
|
|
||||||
return reject(err);
|
|
||||||
return resolve(true);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
async function complete(id, meta) {
|
|
||||||
const query = `UPDATE queue SET completed = CURRENT_TIMESTAMP, meta = ? WHERE id = ?;`;
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
return db.run(query, [meta, id], (err, row) => {
|
|
||||||
if (err)
|
|
||||||
return reject(err);
|
|
||||||
return resolve(true);
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -212,7 +149,7 @@ app.get('/', async (req, res, next) => {
|
||||||
}
|
}
|
||||||
res.send(html);
|
res.send(html);
|
||||||
});
|
});
|
||||||
app.post('/', uploadZip.single('dataset'), async (req, res, next) => {
|
app.post('/', upload.single('dataset'), async (req, res, next) => {
|
||||||
let fileHash;
|
let fileHash;
|
||||||
let filePath;
|
let filePath;
|
||||||
let fileExists;
|
let fileExists;
|
||||||
|
@ -238,7 +175,7 @@ app.post('/', uploadZip.single('dataset'), async (req, res, next) => {
|
||||||
}
|
}
|
||||||
if (!fileExists) {
|
if (!fileExists) {
|
||||||
try {
|
try {
|
||||||
await promises_1.default.copyFile(req.file.path, filePath);
|
await promises_1.default.rename(req.file.path, filePath);
|
||||||
console.log(`Saved dataset with hash ${fileHash}`);
|
console.log(`Saved dataset with hash ${fileHash}`);
|
||||||
}
|
}
|
||||||
catch (err) {
|
catch (err) {
|
||||||
|
@ -248,13 +185,13 @@ app.post('/', uploadZip.single('dataset'), async (req, res, next) => {
|
||||||
}
|
}
|
||||||
else {
|
else {
|
||||||
console.warn(`Dataset with hash ${fileHash} already exists...`);
|
console.warn(`Dataset with hash ${fileHash} already exists...`);
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
await promises_1.default.unlink(req.file.path);
|
await promises_1.default.unlink(req.file.path);
|
||||||
}
|
}
|
||||||
catch (err) {
|
catch (err) {
|
||||||
console.error(err);
|
console.error(err);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
try {
|
try {
|
||||||
id = await add(req.body.email, req.body.name, fileHash, req.body.model);
|
id = await add(req.body.email, req.body.name, fileHash, req.body.model);
|
||||||
}
|
}
|
||||||
|
@ -264,42 +201,6 @@ app.post('/', uploadZip.single('dataset'), async (req, res, next) => {
|
||||||
}
|
}
|
||||||
res.send(`<html><body>Dataset for job ${req.body.name} has been uploaded successfully. You will be emailed when your job has started and when it has completed training. <br /> Monitor job status here: <a href="/job/${id}">${id}</a></body></html>`);
|
res.send(`<html><body>Dataset for job ${req.body.name} has been uploaded successfully. You will be emailed when your job has started and when it has completed training. <br /> Monitor job status here: <a href="/job/${id}">${id}</a></body></html>`);
|
||||||
});
|
});
|
||||||
app.post('/job/:id', uploadOnnx.single('model'), async (req, res, next) => {
|
|
||||||
let filePath;
|
|
||||||
let meta = null;
|
|
||||||
let id;
|
|
||||||
req.setTimeout(0);
|
|
||||||
if (typeof req.file === 'undefined' && req.file === null) {
|
|
||||||
console.error('No file in upload');
|
|
||||||
return next('ERROR: Please model as ONNX file');
|
|
||||||
}
|
|
||||||
filePath = (0, path_1.join)(data, `${id}.onnx`);
|
|
||||||
if (typeof req.body.meta !== 'undefined') {
|
|
||||||
meta = req.body.meta;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
await promises_1.default.copyFile(req.file.path, filePath);
|
|
||||||
console.log(`Saved model for job ${id}`);
|
|
||||||
}
|
|
||||||
catch (err) {
|
|
||||||
console.error(err);
|
|
||||||
return next(err);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
await promises_1.default.unlink(req.file.path);
|
|
||||||
}
|
|
||||||
catch (err) {
|
|
||||||
console.error(err);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
await complete(id, meta);
|
|
||||||
}
|
|
||||||
catch (err) {
|
|
||||||
console.log(err);
|
|
||||||
return next(`Error completing training job ${id}`);
|
|
||||||
}
|
|
||||||
res.json({ id });
|
|
||||||
});
|
|
||||||
app.get('/job/:id', async (req, res, next) => {
|
app.get('/job/:id', async (req, res, next) => {
|
||||||
let jobStatus;
|
let jobStatus;
|
||||||
if (typeof req.params.id === 'undefined' || req.params.id === null) {
|
if (typeof req.params.id === 'undefined' || req.params.id === null) {
|
||||||
|
@ -396,72 +297,18 @@ app.get('/dataset/:id', async (req, res, next) => {
|
||||||
stream.pipe(res);
|
stream.pipe(res);
|
||||||
});
|
});
|
||||||
app.get('/job', async (req, res, next) => {
|
app.get('/job', async (req, res, next) => {
|
||||||
let jobArr;
|
let jobId;
|
||||||
try {
|
try {
|
||||||
jobArr = await job();
|
jobId = await job();
|
||||||
}
|
}
|
||||||
catch (err) {
|
catch (err) {
|
||||||
console.error(err);
|
console.error(err);
|
||||||
return next('Error getting job');
|
return next('Error getting job');
|
||||||
}
|
}
|
||||||
res.json(jobArr);
|
res.json([jobId]);
|
||||||
});
|
|
||||||
app.get('/jobs', async (req, res, next) => {
|
|
||||||
let jobArr;
|
|
||||||
try {
|
|
||||||
jobArr = await jobs();
|
|
||||||
}
|
|
||||||
catch (err) {
|
|
||||||
console.error(err);
|
|
||||||
return next('Error getting job');
|
|
||||||
}
|
|
||||||
res.json(jobArr);
|
|
||||||
});
|
|
||||||
app.post('/job/claim/:id', async (req, res, next) => {
|
|
||||||
let id;
|
|
||||||
let jobObj;
|
|
||||||
let resObj = {};
|
|
||||||
if (typeof req.params.id === 'undefined' || req.params.id === null) {
|
|
||||||
console.error(`No dataset id provided`);
|
|
||||||
return next('Invalid request');
|
|
||||||
}
|
|
||||||
id = req.params.id;
|
|
||||||
try {
|
|
||||||
jobObj = await claim(id);
|
|
||||||
console.log(`Job ${id} was claimed`);
|
|
||||||
}
|
|
||||||
catch (err) {
|
|
||||||
console.error(err);
|
|
||||||
return next('Error claiming job');
|
|
||||||
}
|
|
||||||
resObj.id = id;
|
|
||||||
resObj.path = `/dataset/${id}`;
|
|
||||||
resObj.dataset = jobObj.dataset;
|
|
||||||
resObj.model = jobObj.model;
|
|
||||||
resObj.name = jobObj.name;
|
|
||||||
res.json(resObj);
|
|
||||||
});
|
|
||||||
app.post('/job/fail/:id', async (req, res, next) => {
|
|
||||||
let id;
|
|
||||||
let meta = null;
|
|
||||||
if (typeof req.params.id === 'undefined' || req.params.id === null) {
|
|
||||||
console.error(`No dataset id provided`);
|
|
||||||
return next('Invalid request');
|
|
||||||
}
|
|
||||||
id = req.params.id;
|
|
||||||
if (typeof req.body.meta !== 'undefined') {
|
|
||||||
meta = req.body.meta;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
await fail(id, meta);
|
|
||||||
console.log(`Job ${id} failed`);
|
|
||||||
}
|
|
||||||
catch (err) {
|
|
||||||
console.error(err);
|
|
||||||
return next('Error failing job');
|
|
||||||
}
|
|
||||||
res.json(true);
|
|
||||||
});
|
});
|
||||||
|
//app.get('/jobs');
|
||||||
|
//app.post('/job/started/:id', )
|
||||||
app.listen(port, () => {
|
app.listen(port, () => {
|
||||||
console.log(`yolo_web running on port ${port}`);
|
console.log(`yolo_web running on port ${port}`);
|
||||||
});
|
});
|
||||||
|
|
File diff suppressed because one or more lines are too long
|
@ -1,9 +0,0 @@
|
||||||
services:
|
|
||||||
yolo_web:
|
|
||||||
image: yolo_web
|
|
||||||
container_name: yolo_web
|
|
||||||
volumes:
|
|
||||||
- './views:/code/views'
|
|
||||||
- './data/:/code/data'
|
|
||||||
ports:
|
|
||||||
- '3333:3333'
|
|
|
@ -1,24 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
if [ -f .env ]; then
|
|
||||||
source .env
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z ${1} ]; then
|
|
||||||
echo "Please provide a job"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
JOB="${1}"
|
|
||||||
|
|
||||||
JSON=$(curl -s -X POST "${YOLO_WEB_URL}/job/claim/${JOB}")
|
|
||||||
|
|
||||||
if [[ "${JSON}" != *"{"* ]]; then
|
|
||||||
echo ERROR
|
|
||||||
echo "${JSON}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
bash ./scripts/train.sh "${JSON}"
|
|
|
@ -1,20 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
if [ -f .env ]; then
|
|
||||||
source .env
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z ${YOLO_WEB_URL} ]; then
|
|
||||||
YOLO_WEB_URL=http://localhost:3333
|
|
||||||
fi
|
|
||||||
|
|
||||||
JOB_JSON=$(curl -s "${YOLO_WEB_URL}/job")
|
|
||||||
JOB=$(echo $JOB_JSON | jq -r '.[0]')
|
|
||||||
|
|
||||||
if [[ "${JOB}" != "null" ]]; then
|
|
||||||
bash ./scripts/claim.sh "${JOB}"
|
|
||||||
else
|
|
||||||
echo "No jobs"
|
|
||||||
fi
|
|
|
@ -1,47 +0,0 @@
|
||||||
set -e
|
|
||||||
|
|
||||||
if [ -f .env ]; then
|
|
||||||
source .env
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z ${1} ]; then
|
|
||||||
echo "Please provide a JSON object"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
function fail () {
|
|
||||||
ID="${1}"
|
|
||||||
META=${2}
|
|
||||||
URL="${YOLO_WEB_URL}/job/fail/${ID}"
|
|
||||||
echo "${META}"
|
|
||||||
curl -s -X POST \
|
|
||||||
-H 'Content-Type: application/json' \
|
|
||||||
-d "{\"meta\":\"${META}\"}" "${URL}"
|
|
||||||
}
|
|
||||||
|
|
||||||
JSON="${1}"
|
|
||||||
|
|
||||||
ID=$(echo $JSON | jq -r '.id')
|
|
||||||
DATASET=$(echo $JSON | jq -r '.dataset')
|
|
||||||
MODEL=$(echo $JSON | jq -r '.model')
|
|
||||||
NAME=$(echo $JSON | jq -r '.name')
|
|
||||||
FILEPATH=$(echo $JSON | jq -r '.path')
|
|
||||||
|
|
||||||
DOWNLOAD="${YOLO_WEB_URL}${FILEPATH}"
|
|
||||||
DEST="${YOLOv5}${DATASET}.zip"
|
|
||||||
UNZIPPED="${YOLOv5}${DATASET}"
|
|
||||||
|
|
||||||
echo "Downloading ${DOWNLOAD}"
|
|
||||||
|
|
||||||
if [ ! -f "${DEST}" ]; then
|
|
||||||
wget -q -O "${DEST}" "${DOWNLOAD}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
unzip "${DEST}" -d "${UNZIPPED}"
|
|
||||||
|
|
||||||
if [ ! -f "${UNZIPPED}/data.yaml" ]; then
|
|
||||||
fail "${ID}" "Invalid dataset"
|
|
||||||
rm -rf "${UNZIPPED}"
|
|
||||||
rm "${DEST}"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
145
src/index.ts
145
src/index.ts
|
@ -38,8 +38,7 @@ function fileFilter (req: any, file: any, cb: any) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const uploadZip : any = multer({ storage, fileFilter });
|
const upload : any = multer({ storage, fileFilter });
|
||||||
const uploadOnnx : any = multer({ storage });
|
|
||||||
|
|
||||||
app.use(bodyParser.json());
|
app.use(bodyParser.json());
|
||||||
app.use(bodyParser.urlencoded({ extended: true }));
|
app.use(bodyParser.urlencoded({ extended: true }));
|
||||||
|
@ -83,18 +82,12 @@ async function status (id : string) : Promise<string> {
|
||||||
return new Promise((resolve : Function, reject : Function) => {
|
return new Promise((resolve : Function, reject : Function) => {
|
||||||
return db.all(query, [id], (err : Error, rows : any) => {
|
return db.all(query, [id], (err : Error, rows : any) => {
|
||||||
if (err) return reject(err);
|
if (err) return reject(err);
|
||||||
if (rows.length !== 1) {
|
|
||||||
return resolve(jobStatus);
|
|
||||||
}
|
|
||||||
const obj : any = rows[0];
|
|
||||||
if (rows[0].started === null) {
|
if (rows[0].started === null) {
|
||||||
jobStatus = `Has not started`
|
jobStatus = `Has not started`
|
||||||
} else if (rows[0].failed !== null) {
|
} else if (rows[0].failed !== null) {
|
||||||
jobStatus = `Failed <br /> <pre>${rows[0].meta}</pre>`;
|
jobStatus = `Failed <br /> <pre>${rows[0].meta}</pre>`;
|
||||||
} else if (rows[0].completed !== null) {
|
} else if (rows[0].completed !== null) {
|
||||||
jobStatus = `Completed ${rows[0].completed} <a href="/model/${rows[0].model}/${id}">Download</a>`;
|
jobStatus = `Completed ${rows[0].completed} <a href="/model/${rows[0].model}/${id}">Download</a>`;
|
||||||
} else if (rows[0].started !== null) {
|
|
||||||
jobStatus = `Started ${rows[0].started}`;
|
|
||||||
}
|
}
|
||||||
console.log(`Got status for job ${id}: ${jobStatus}`);
|
console.log(`Got status for job ${id}: ${jobStatus}`);
|
||||||
return resolve(jobStatus);
|
return resolve(jobStatus);
|
||||||
|
@ -128,7 +121,7 @@ async function dataset (id : string) : Promise<string> {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async function job () : Promise<string[]> {
|
async function job () : Promise<string|null> {
|
||||||
const query : string = `SELECT id FROM queue WHERE
|
const query : string = `SELECT id FROM queue WHERE
|
||||||
started IS NULL
|
started IS NULL
|
||||||
AND completed IS NULL
|
AND completed IS NULL
|
||||||
|
@ -139,23 +132,9 @@ async function job () : Promise<string[]> {
|
||||||
return db.all(query, [], (err : Error, rows : any) => {
|
return db.all(query, [], (err : Error, rows : any) => {
|
||||||
if (err) return reject(err);
|
if (err) return reject(err);
|
||||||
if (rows.length < 1) {
|
if (rows.length < 1) {
|
||||||
return resolve([]);
|
return resolve(null);
|
||||||
}
|
}
|
||||||
return resolve([rows[0].id]);
|
return resolve(rows[0].id);
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async function jobs () : Promise<string[]> {
|
|
||||||
const query : string = `SELECT id FROM queue WHERE
|
|
||||||
started IS NULL
|
|
||||||
AND completed IS NULL
|
|
||||||
AND failed IS NULL
|
|
||||||
ORDER BY created ASC;`;
|
|
||||||
return new Promise((resolve : Function, reject : Function) => {
|
|
||||||
return db.all(query, [], (err : Error, rows : any) => {
|
|
||||||
if (err) return reject(err);
|
|
||||||
return resolve(rows.map((el : any) => el.id));
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
@ -171,7 +150,7 @@ async function claim (id : string) : Promise<string> {
|
||||||
if (rows[0].started !== null) {
|
if (rows[0].started !== null) {
|
||||||
return reject(new Error(`Job ${id} is already claimed`));
|
return reject(new Error(`Job ${id} is already claimed`));
|
||||||
}
|
}
|
||||||
const claimQuery : string = `UPDATE queue SET started = CURRENT_TIMESTAMP WHERE id = ?;`;
|
const claimQuery : string = `UPDATE queue SET started = CURRENT_TIMESTAMP WHERE id = ? LIMIT 1;`;
|
||||||
return db.run(claimQuery, [id], (err : Error, row : any) => {
|
return db.run(claimQuery, [id], (err : Error, row : any) => {
|
||||||
if (err) return reject(err);
|
if (err) return reject(err);
|
||||||
return resolve(rows[0]);
|
return resolve(rows[0]);
|
||||||
|
@ -180,20 +159,10 @@ async function claim (id : string) : Promise<string> {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fail (id : string, meta : string | null) : Promise<boolean> {
|
async function fail (id : string, meta : string) : Promise<boolean> {
|
||||||
const query : string = `UPDATE queue SET failed = CURRENT_TIMESTAMP, meta = ? WHERE id = ?;`;
|
const query : string = `UPDATE queue SET failed = CURRENT_TIMESTAMP WHERE id = ? LIMIT 1;`;
|
||||||
return new Promise((resolve : Function, reject : Function) => {
|
return new Promise((resolve : Function, reject : Function) => {
|
||||||
return db.run(query, [ meta, id], (err : Error, row : any) => {
|
return db.run(query, [ id ], (err : Error, row : any) => {
|
||||||
if (err) return reject(err);
|
|
||||||
return resolve(true);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async function complete (id : string, meta : string | null) : Promise<boolean> {
|
|
||||||
const query : string = `UPDATE queue SET completed = CURRENT_TIMESTAMP, meta = ? WHERE id = ?;`;
|
|
||||||
return new Promise((resolve : Function, reject : Function) => {
|
|
||||||
return db.run(query, [ meta, id ], (err : Error, row : any) => {
|
|
||||||
if (err) return reject(err);
|
if (err) return reject(err);
|
||||||
return resolve(true);
|
return resolve(true);
|
||||||
});
|
});
|
||||||
|
@ -211,7 +180,7 @@ app.get('/', async (req : Request, res : Response, next : NextFunction) => {
|
||||||
res.send(html);
|
res.send(html);
|
||||||
});
|
});
|
||||||
|
|
||||||
app.post('/', uploadZip.single('dataset'), async (req : Request, res : Response, next : NextFunction) => {
|
app.post('/', upload.single('dataset'), async (req : Request, res : Response, next : NextFunction) => {
|
||||||
let fileHash : string;
|
let fileHash : string;
|
||||||
let filePath : string;
|
let filePath : string;
|
||||||
let fileExists : boolean;
|
let fileExists : boolean;
|
||||||
|
@ -240,21 +209,21 @@ app.post('/', uploadZip.single('dataset'), async (req : Request, res : Response,
|
||||||
|
|
||||||
if (!fileExists) {
|
if (!fileExists) {
|
||||||
try {
|
try {
|
||||||
await fs.copyFile(req.file.path, filePath);
|
await fs.rename(req.file.path, filePath);
|
||||||
console.log(`Saved dataset with hash ${fileHash}`);
|
console.log(`Saved dataset with hash ${fileHash}`);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error(err);
|
console.error(err);
|
||||||
return next(err);
|
return next(err);
|
||||||
}
|
}
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
console.warn(`Dataset with hash ${fileHash} already exists...`);
|
console.warn(`Dataset with hash ${fileHash} already exists...`);
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await fs.unlink(req.file.path);
|
await fs.unlink(req.file.path);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error(err);
|
console.error(err);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
id = await add(req.body.email, req.body.name, fileHash, req.body.model);
|
id = await add(req.body.email, req.body.name, fileHash, req.body.model);
|
||||||
|
@ -266,47 +235,6 @@ app.post('/', uploadZip.single('dataset'), async (req : Request, res : Response,
|
||||||
res.send(`<html><body>Dataset for job ${req.body.name} has been uploaded successfully. You will be emailed when your job has started and when it has completed training. <br /> Monitor job status here: <a href="/job/${id}">${id}</a></body></html>`);
|
res.send(`<html><body>Dataset for job ${req.body.name} has been uploaded successfully. You will be emailed when your job has started and when it has completed training. <br /> Monitor job status here: <a href="/job/${id}">${id}</a></body></html>`);
|
||||||
});
|
});
|
||||||
|
|
||||||
app.post('/job/:id', uploadOnnx.single('model'), async (req : Request, res : Response, next : NextFunction) => {
|
|
||||||
let filePath : string;
|
|
||||||
let meta : string = null;
|
|
||||||
let id : string;
|
|
||||||
|
|
||||||
req.setTimeout(0);
|
|
||||||
|
|
||||||
if (typeof req.file === 'undefined' && req.file === null) {
|
|
||||||
console.error('No file in upload');
|
|
||||||
return next('ERROR: Please model as ONNX file');
|
|
||||||
}
|
|
||||||
|
|
||||||
filePath = join(data, `${id}.onnx`);
|
|
||||||
if (typeof req.body.meta !== 'undefined') {
|
|
||||||
meta = req.body.meta;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
await fs.copyFile(req.file.path, filePath);
|
|
||||||
console.log(`Saved model for job ${id}`);
|
|
||||||
} catch (err) {
|
|
||||||
console.error(err);
|
|
||||||
return next(err);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
await fs.unlink(req.file.path);
|
|
||||||
} catch (err) {
|
|
||||||
console.error(err);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
await complete(id, meta);
|
|
||||||
} catch (err) {
|
|
||||||
console.log(err);
|
|
||||||
return next(`Error completing training job ${id}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
res.json({ id });
|
|
||||||
});
|
|
||||||
|
|
||||||
app.get('/job/:id', async (req : Request, res : Response, next : NextFunction) => {
|
app.get('/job/:id', async (req : Request, res : Response, next : NextFunction) => {
|
||||||
let jobStatus : string;
|
let jobStatus : string;
|
||||||
|
|
||||||
|
@ -416,29 +344,16 @@ app.get('/dataset/:id', async (req : Request, res: Response, next : NextFunction
|
||||||
});
|
});
|
||||||
|
|
||||||
app.get('/job', async (req : Request, res: Response, next : NextFunction) => {
|
app.get('/job', async (req : Request, res: Response, next : NextFunction) => {
|
||||||
let jobArr : string[];
|
let jobId : string;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
jobArr = await job();
|
jobId = await job();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error(err);
|
console.error(err);
|
||||||
return next('Error getting job');
|
return next('Error getting job');
|
||||||
}
|
}
|
||||||
|
|
||||||
res.json(jobArr);
|
res.json([jobId]);
|
||||||
});
|
|
||||||
|
|
||||||
app.get('/jobs', async (req : Request, res: Response, next : NextFunction) => {
|
|
||||||
let jobArr : string[];
|
|
||||||
|
|
||||||
try {
|
|
||||||
jobArr = await jobs();
|
|
||||||
} catch (err) {
|
|
||||||
console.error(err);
|
|
||||||
return next('Error getting job');
|
|
||||||
}
|
|
||||||
|
|
||||||
res.json(jobArr);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
app.post('/job/claim/:id', async (req : Request, res: Response, next : NextFunction) => {
|
app.post('/job/claim/:id', async (req : Request, res: Response, next : NextFunction) => {
|
||||||
|
@ -455,24 +370,22 @@ app.post('/job/claim/:id', async (req : Request, res: Response, next : NextFunct
|
||||||
|
|
||||||
try {
|
try {
|
||||||
jobObj = await claim(id);
|
jobObj = await claim(id);
|
||||||
console.log(`Job ${id} was claimed`);
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error(err);
|
console.error(err);
|
||||||
return next('Error claiming job');
|
return next('Error claiming job');
|
||||||
}
|
}
|
||||||
|
|
||||||
resObj.id = id;
|
resJob.id = id;
|
||||||
resObj.path = `/dataset/${id}`;
|
resJob.datasetPath = `/dataset/${id}`;
|
||||||
resObj.dataset = jobObj.dataset;
|
resJob.model = jobObj.model;
|
||||||
resObj.model = jobObj.model;
|
|
||||||
resObj.name = jobObj.name;
|
|
||||||
|
|
||||||
res.json(resObj);
|
res.json(resJob);
|
||||||
});
|
});
|
||||||
|
|
||||||
app.post('/job/fail/:id', async (req : Request, res: Response, next : NextFunction) => {
|
app.post('/job/fail/:id', async (req : Request, res: Response, next : NextFunction) => {
|
||||||
let id : string;
|
let id : string;
|
||||||
let meta : string = null;
|
let jobObj : any;
|
||||||
|
let resObj : any = {};
|
||||||
|
|
||||||
if (typeof req.params.id === 'undefined' || req.params.id === null) {
|
if (typeof req.params.id === 'undefined' || req.params.id === null) {
|
||||||
console.error(`No dataset id provided`);
|
console.error(`No dataset id provided`);
|
||||||
|
@ -480,20 +393,22 @@ app.post('/job/fail/:id', async (req : Request, res: Response, next : NextFuncti
|
||||||
}
|
}
|
||||||
|
|
||||||
id = req.params.id;
|
id = req.params.id;
|
||||||
if (typeof req.body.meta !== 'undefined') {
|
|
||||||
meta = req.body.meta;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await fail(id, meta);
|
jobObj = await claim(id);
|
||||||
console.log(`Job ${id} failed`);
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error(err);
|
console.error(err);
|
||||||
return next('Error failing job');
|
return next('Error claiming job');
|
||||||
}
|
}
|
||||||
|
|
||||||
res.json(true);
|
resJob.id = id;
|
||||||
|
resJob.datasetPath = `/dataset/${id}`;
|
||||||
|
resJob.model = jobObj.model;
|
||||||
|
|
||||||
|
res.json(resJob);
|
||||||
});
|
});
|
||||||
|
//app.get('/jobs');
|
||||||
|
//app.post('/job/started/:id', )
|
||||||
|
|
||||||
app.listen(port, () => {
|
app.listen(port, () => {
|
||||||
console.log(`yolo_web running on port ${port}`);
|
console.log(`yolo_web running on port ${port}`);
|
||||||
|
|
Loading…
Reference in New Issue