mattmcw
/
yolo_web
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: mattmcw:82a7d598fea2cf419893b39cb8d3a0e128d75a0d
Branches Tags
mattmcw:main
..
compare: mattmcw:eea288b28d2ea31bb37b3835fac8f5059571e36c
Branches Tags
mattmcw:main

No commits in common. "82a7d598fea2cf419893b39cb8d3a0e128d75a0d" and "eea288b28d2ea31bb37b3835fac8f5059571e36c" have entirely different histories.
82a7d598fe ... eea288b28d

10 changed files with 52 additions and 394 deletions
Show Stats Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,3 +1,2 @@
node_modules node_modules
data/* data/*
.env

3
Dockerfile
View File

@ -2,9 +2,8 @@ FROM node:lts-alpine
WORKDIR /code WORKDIR /code
COPY ./dist /code/dist
COPY ./package*.json /code/ COPY ./package*.json /code/
RUN npm install --only-production RUN npm install --only-production
COPY ./dist /code/dist
CMD ["npm", "run", "start"] CMD ["npm", "run", "start"]

2
default.env
View File

@ -1,2 +0,0 @@
YOLO_WEB_URL=http://localhost:3333
YOLOv5=../yolo_train/

185
dist/index.js vendored
View File

@ -38,8 +38,7 @@ function fileFilter(req, file, cb) {
cb(new Error("Dataset is not of type zip"), false); cb(new Error("Dataset is not of type zip"), false);
} }
} }
const uploadZip = (0, multer_1.default)({ storage, fileFilter }); const upload = (0, multer_1.default)({ storage, fileFilter });
const uploadOnnx = (0, multer_1.default)({ storage });
app.use(body_parser_1.default.json()); app.use(body_parser_1.default.json());
app.use(body_parser_1.default.urlencoded({ extended: true })); app.use(body_parser_1.default.urlencoded({ extended: true }));
function hash(path) { function hash(path) {
@ -81,10 +80,6 @@ async function status(id) {
return db.all(query, [id], (err, rows) => { return db.all(query, [id], (err, rows) => {
if (err) if (err)
return reject(err); return reject(err);
if (rows.length !== 1) {
return resolve(jobStatus);
}
const obj = rows[0];
if (rows[0].started === null) { if (rows[0].started === null) {
jobStatus = `Has not started`; jobStatus = `Has not started`;
} }
@ -94,9 +89,6 @@ async function status(id) {
else if (rows[0].completed !== null) { else if (rows[0].completed !== null) {
jobStatus = `Completed ${rows[0].completed} <a href="/model/${rows[0].model}/${id}">Download</a>`; jobStatus = `Completed ${rows[0].completed} <a href="/model/${rows[0].model}/${id}">Download</a>`;
} }
else if (rows[0].started !== null) {
jobStatus = `Started ${rows[0].started}`;
}
console.log(`Got status for job ${id}: ${jobStatus}`); console.log(`Got status for job ${id}: ${jobStatus}`);
return resolve(jobStatus); return resolve(jobStatus);
}); });
@ -140,64 +132,9 @@ async function job() {
if (err) if (err)
return reject(err); return reject(err);
if (rows.length < 1) { if (rows.length < 1) {
return resolve([]); return resolve(null);
} }
return resolve([rows[0].id]); return resolve(rows[0].id);
});
});
}
async function jobs() {
const query = `SELECT id FROM queue WHERE
started IS NULL
AND completed IS NULL
AND failed IS NULL
ORDER BY created ASC;`;
return new Promise((resolve, reject) => {
return db.all(query, [], (err, rows) => {
if (err)
return reject(err);
return resolve(rows.map((el) => el.id));
});
});
}
async function claim(id) {
const query = `SELECT * FROM queue WHERE id = ? LIMIT 1;`;
return new Promise((resolve, reject) => {
return db.all(query, [id], (err, rows) => {
if (err)
return reject(err);
if (rows.length < 1) {
return reject(new Error(`Dataset ${id} does not exist`));
}
if (rows[0].started !== null) {
return reject(new Error(`Job ${id} is already claimed`));
}
const claimQuery = `UPDATE queue SET started = CURRENT_TIMESTAMP WHERE id = ?;`;
return db.run(claimQuery, [id], (err, row) => {
if (err)
return reject(err);
return resolve(rows[0]);
});
});
});
}
async function fail(id, meta) {
const query = `UPDATE queue SET failed = CURRENT_TIMESTAMP, meta = ? WHERE id = ?;`;
return new Promise((resolve, reject) => {
return db.run(query, [meta, id], (err, row) => {
if (err)
return reject(err);
return resolve(true);
});
});
}
async function complete(id, meta) {
const query = `UPDATE queue SET completed = CURRENT_TIMESTAMP, meta = ? WHERE id = ?;`;
return new Promise((resolve, reject) => {
return db.run(query, [meta, id], (err, row) => {
if (err)
return reject(err);
return resolve(true);
}); });
}); });
} }
@ -212,7 +149,7 @@ app.get('/', async (req, res, next) => {
} }
res.send(html); res.send(html);
}); });
app.post('/', uploadZip.single('dataset'), async (req, res, next) => { app.post('/', upload.single('dataset'), async (req, res, next) => {
let fileHash; let fileHash;
let filePath; let filePath;
let fileExists; let fileExists;
@ -238,7 +175,7 @@ app.post('/', uploadZip.single('dataset'), async (req, res, next) => {
} }
if (!fileExists) { if (!fileExists) {
try { try {
await promises_1.default.copyFile(req.file.path, filePath); await promises_1.default.rename(req.file.path, filePath);
console.log(`Saved dataset with hash ${fileHash}`); console.log(`Saved dataset with hash ${fileHash}`);
} }
catch (err) { catch (err) {
@ -248,12 +185,12 @@ app.post('/', uploadZip.single('dataset'), async (req, res, next) => {
} }
else { else {
console.warn(`Dataset with hash ${fileHash} already exists...`); console.warn(`Dataset with hash ${fileHash} already exists...`);
} try {
try { await promises_1.default.unlink(req.file.path);
await promises_1.default.unlink(req.file.path); }
} catch (err) {
catch (err) { console.error(err);
console.error(err); }
} }
try { try {
id = await add(req.body.email, req.body.name, fileHash, req.body.model); id = await add(req.body.email, req.body.name, fileHash, req.body.model);
@ -264,42 +201,6 @@ app.post('/', uploadZip.single('dataset'), async (req, res, next) => {
} }
res.send(`<html><body>Dataset for job ${req.body.name} has been uploaded successfully. You will be emailed when your job has started and when it has completed training. <br /> Monitor job status here: <a href="/job/${id}">${id}</a></body></html>`); res.send(`<html><body>Dataset for job ${req.body.name} has been uploaded successfully. You will be emailed when your job has started and when it has completed training. <br /> Monitor job status here: <a href="/job/${id}">${id}</a></body></html>`);
}); });
app.post('/job/:id', uploadOnnx.single('model'), async (req, res, next) => {
let filePath;
let meta = null;
let id;
req.setTimeout(0);
if (typeof req.file === 'undefined' && req.file === null) {
console.error('No file in upload');
return next('ERROR: Please model as ONNX file');
}
filePath = (0, path_1.join)(data, `${id}.onnx`);
if (typeof req.body.meta !== 'undefined') {
meta = req.body.meta;
}
try {
await promises_1.default.copyFile(req.file.path, filePath);
console.log(`Saved model for job ${id}`);
}
catch (err) {
console.error(err);
return next(err);
}
try {
await promises_1.default.unlink(req.file.path);
}
catch (err) {
console.error(err);
}
try {
await complete(id, meta);
}
catch (err) {
console.log(err);
return next(`Error completing training job ${id}`);
}
res.json({ id });
});
app.get('/job/:id', async (req, res, next) => { app.get('/job/:id', async (req, res, next) => {
let jobStatus; let jobStatus;
if (typeof req.params.id === 'undefined' || req.params.id === null) { if (typeof req.params.id === 'undefined' || req.params.id === null) {
@ -396,72 +297,18 @@ app.get('/dataset/:id', async (req, res, next) => {
stream.pipe(res); stream.pipe(res);
}); });
app.get('/job', async (req, res, next) => { app.get('/job', async (req, res, next) => {
let jobArr; let jobId;
try { try {
jobArr = await job(); jobId = await job();
} }
catch (err) { catch (err) {
console.error(err); console.error(err);
return next('Error getting job'); return next('Error getting job');
} }
res.json(jobArr); res.json([jobId]);
});
app.get('/jobs', async (req, res, next) => {
let jobArr;
try {
jobArr = await jobs();
}
catch (err) {
console.error(err);
return next('Error getting job');
}
res.json(jobArr);
});
app.post('/job/claim/:id', async (req, res, next) => {
let id;
let jobObj;
let resObj = {};
if (typeof req.params.id === 'undefined' || req.params.id === null) {
console.error(`No dataset id provided`);
return next('Invalid request');
}
id = req.params.id;
try {
jobObj = await claim(id);
console.log(`Job ${id} was claimed`);
}
catch (err) {
console.error(err);
return next('Error claiming job');
}
resObj.id = id;
resObj.path = `/dataset/${id}`;
resObj.dataset = jobObj.dataset;
resObj.model = jobObj.model;
resObj.name = jobObj.name;
res.json(resObj);
});
app.post('/job/fail/:id', async (req, res, next) => {
let id;
let meta = null;
if (typeof req.params.id === 'undefined' || req.params.id === null) {
console.error(`No dataset id provided`);
return next('Invalid request');
}
id = req.params.id;
if (typeof req.body.meta !== 'undefined') {
meta = req.body.meta;
}
try {
await fail(id, meta);
console.log(`Job ${id} failed`);
}
catch (err) {
console.error(err);
return next('Error failing job');
}
res.json(true);
}); });
//app.get('/jobs');
//app.post('/job/started/:id', )
app.listen(port, () => { app.listen(port, () => {
console.log(`yolo_web running on port ${port}`); console.log(`yolo_web running on port ${port}`);
}); });

2
dist/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

9
docker-compose.yaml
View File

@ -1,9 +0,0 @@
services:
yolo_web:
image: yolo_web
container_name: yolo_web
volumes:
- './views:/code/views'
- './data/:/code/data'
ports:
- '3333:3333'

24
scripts/claim.sh
View File

@ -1,24 +0,0 @@
#!/bin/bash
set -e
if [ -f .env ]; then
source .env
fi
if [ -z ${1} ]; then
echo "Please provide a job"
exit 1
fi
JOB="${1}"
JSON=$(curl -s -X POST "${YOLO_WEB_URL}/job/claim/${JOB}")
if [[ "${JSON}" != *"{"* ]]; then
echo ERROR
echo "${JSON}"
exit 1
fi
bash ./scripts/train.sh "${JSON}"

20
scripts/job.sh
View File

@ -1,20 +0,0 @@
#!/bin/bash
set -e
if [ -f .env ]; then
source .env
fi
if [ -z ${YOLO_WEB_URL} ]; then
YOLO_WEB_URL=http://localhost:3333
fi
JOB_JSON=$(curl -s "${YOLO_WEB_URL}/job")
JOB=$(echo $JOB_JSON | jq -r '.[0]')
if [[ "${JOB}" != "null" ]]; then
bash ./scripts/claim.sh "${JOB}"
else
echo "No jobs"
fi

47
scripts/train.sh
View File

@ -1,47 +0,0 @@
set -e
if [ -f .env ]; then
source .env
fi
if [ -z ${1} ]; then
echo "Please provide a JSON object"
exit 1
fi
function fail () {
ID="${1}"
META=${2}
URL="${YOLO_WEB_URL}/job/fail/${ID}"
echo "${META}"
curl -s -X POST \
-H 'Content-Type: application/json' \
-d "{\"meta\":\"${META}\"}" "${URL}"
}
JSON="${1}"
ID=$(echo $JSON | jq -r '.id')
DATASET=$(echo $JSON | jq -r '.dataset')
MODEL=$(echo $JSON | jq -r '.model')
NAME=$(echo $JSON | jq -r '.name')
FILEPATH=$(echo $JSON | jq -r '.path')
DOWNLOAD="${YOLO_WEB_URL}${FILEPATH}"
DEST="${YOLOv5}${DATASET}.zip"
UNZIPPED="${YOLOv5}${DATASET}"
echo "Downloading ${DOWNLOAD}"
if [ ! -f "${DEST}" ]; then
wget -q -O "${DEST}" "${DOWNLOAD}"
fi
unzip "${DEST}" -d "${UNZIPPED}"
if [ ! -f "${UNZIPPED}/data.yaml" ]; then
fail "${ID}" "Invalid dataset"
rm -rf "${UNZIPPED}"
rm "${DEST}"
exit 1
fi

153
src/index.ts
View File

@ -38,8 +38,7 @@ function fileFilter (req: any, file: any, cb: any) {
} }
} }
const uploadZip : any = multer({ storage, fileFilter }); const upload : any = multer({ storage, fileFilter });
const uploadOnnx : any = multer({ storage });
app.use(bodyParser.json()); app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true })); app.use(bodyParser.urlencoded({ extended: true }));
@ -83,18 +82,12 @@ async function status (id : string) : Promise<string> {
return new Promise((resolve : Function, reject : Function) => { return new Promise((resolve : Function, reject : Function) => {
return db.all(query, [id], (err : Error, rows : any) => { return db.all(query, [id], (err : Error, rows : any) => {
if (err) return reject(err); if (err) return reject(err);
if (rows.length !== 1) {
return resolve(jobStatus);
}
const obj : any = rows[0];
if (rows[0].started === null) { if (rows[0].started === null) {
jobStatus = `Has not started` jobStatus = `Has not started`
} else if (rows[0].failed !== null) { } else if (rows[0].failed !== null) {
jobStatus = `Failed <br /> <pre>${rows[0].meta}</pre>`; jobStatus = `Failed <br /> <pre>${rows[0].meta}</pre>`;
} else if (rows[0].completed !== null) { } else if (rows[0].completed !== null) {
jobStatus = `Completed ${rows[0].completed} <a href="/model/${rows[0].model}/${id}">Download</a>`; jobStatus = `Completed ${rows[0].completed} <a href="/model/${rows[0].model}/${id}">Download</a>`;
} else if (rows[0].started !== null) {
jobStatus = `Started ${rows[0].started}`;
} }
console.log(`Got status for job ${id}: ${jobStatus}`); console.log(`Got status for job ${id}: ${jobStatus}`);
return resolve(jobStatus); return resolve(jobStatus);
@ -128,7 +121,7 @@ async function dataset (id : string) : Promise<string> {
}); });
} }
async function job () : Promise<string[]> { async function job () : Promise<string|null> {
const query : string = `SELECT id FROM queue WHERE const query : string = `SELECT id FROM queue WHERE
started IS NULL started IS NULL
AND completed IS NULL AND completed IS NULL
@ -139,23 +132,9 @@ async function job () : Promise<string[]> {
return db.all(query, [], (err : Error, rows : any) => { return db.all(query, [], (err : Error, rows : any) => {
if (err) return reject(err); if (err) return reject(err);
if (rows.length < 1) { if (rows.length < 1) {
return resolve([]); return resolve(null);
} }
return resolve([rows[0].id]); return resolve(rows[0].id);
});
});
}
async function jobs () : Promise<string[]> {
const query : string = `SELECT id FROM queue WHERE
started IS NULL
AND completed IS NULL
AND failed IS NULL
ORDER BY created ASC;`;
return new Promise((resolve : Function, reject : Function) => {
return db.all(query, [], (err : Error, rows : any) => {
if (err) return reject(err);
return resolve(rows.map((el : any) => el.id));
}); });
}); });
} }
@ -171,7 +150,7 @@ async function claim (id : string) : Promise<string> {
if (rows[0].started !== null) { if (rows[0].started !== null) {
return reject(new Error(`Job ${id} is already claimed`)); return reject(new Error(`Job ${id} is already claimed`));
} }
const claimQuery : string = `UPDATE queue SET started = CURRENT_TIMESTAMP WHERE id = ?;`; const claimQuery : string = `UPDATE queue SET started = CURRENT_TIMESTAMP WHERE id = ? LIMIT 1;`;
return db.run(claimQuery, [id], (err : Error, row : any) => { return db.run(claimQuery, [id], (err : Error, row : any) => {
if (err) return reject(err); if (err) return reject(err);
return resolve(rows[0]); return resolve(rows[0]);
@ -180,20 +159,10 @@ async function claim (id : string) : Promise<string> {
}); });
} }
async function fail (id : string, meta : string | null) : Promise<boolean> { async function fail (id : string, meta : string) : Promise<boolean> {
const query : string = `UPDATE queue SET failed = CURRENT_TIMESTAMP, meta = ? WHERE id = ?;`; const query : string = `UPDATE queue SET failed = CURRENT_TIMESTAMP WHERE id = ? LIMIT 1;`;
return new Promise((resolve : Function, reject : Function) => { return new Promise((resolve : Function, reject : Function) => {
return db.run(query, [ meta, id], (err : Error, row : any) => { return db.run(query, [ id ], (err : Error, row : any) => {
if (err) return reject(err);
return resolve(true);
});
});
}
async function complete (id : string, meta : string | null) : Promise<boolean> {
const query : string = `UPDATE queue SET completed = CURRENT_TIMESTAMP, meta = ? WHERE id = ?;`;
return new Promise((resolve : Function, reject : Function) => {
return db.run(query, [ meta, id ], (err : Error, row : any) => {
if (err) return reject(err); if (err) return reject(err);
return resolve(true); return resolve(true);
}); });
@ -211,7 +180,7 @@ app.get('/', async (req : Request, res : Response, next : NextFunction) => {
res.send(html); res.send(html);
}); });
app.post('/', uploadZip.single('dataset'), async (req : Request, res : Response, next : NextFunction) => { app.post('/', upload.single('dataset'), async (req : Request, res : Response, next : NextFunction) => {
let fileHash : string; let fileHash : string;
let filePath : string; let filePath : string;
let fileExists : boolean; let fileExists : boolean;
@ -240,20 +209,20 @@ app.post('/', uploadZip.single('dataset'), async (req : Request, res : Response,
if (!fileExists) { if (!fileExists) {
try { try {
await fs.copyFile(req.file.path, filePath); await fs.rename(req.file.path, filePath);
console.log(`Saved dataset with hash ${fileHash}`); console.log(`Saved dataset with hash ${fileHash}`);
} catch (err) { } catch (err) {
console.error(err); console.error(err);
return next(err); return next(err);
} }
} else { } else {
console.warn(`Dataset with hash ${fileHash} already exists...`); console.warn(`Dataset with hash ${fileHash} already exists...`);
} try {
await fs.unlink(req.file.path);
try { } catch (err) {
await fs.unlink(req.file.path); console.error(err);
} catch (err) { }
console.error(err);
} }
try { try {
@ -266,47 +235,6 @@ app.post('/', uploadZip.single('dataset'), async (req : Request, res : Response,
res.send(`<html><body>Dataset for job ${req.body.name} has been uploaded successfully. You will be emailed when your job has started and when it has completed training. <br /> Monitor job status here: <a href="/job/${id}">${id}</a></body></html>`); res.send(`<html><body>Dataset for job ${req.body.name} has been uploaded successfully. You will be emailed when your job has started and when it has completed training. <br /> Monitor job status here: <a href="/job/${id}">${id}</a></body></html>`);
}); });
app.post('/job/:id', uploadOnnx.single('model'), async (req : Request, res : Response, next : NextFunction) => {
let filePath : string;
let meta : string = null;
let id : string;
req.setTimeout(0);
if (typeof req.file === 'undefined' && req.file === null) {
console.error('No file in upload');
return next('ERROR: Please model as ONNX file');
}
filePath = join(data, `${id}.onnx`);
if (typeof req.body.meta !== 'undefined') {
meta = req.body.meta;
}
try {
await fs.copyFile(req.file.path, filePath);
console.log(`Saved model for job ${id}`);
} catch (err) {
console.error(err);
return next(err);
}
try {
await fs.unlink(req.file.path);
} catch (err) {
console.error(err);
}
try {
await complete(id, meta);
} catch (err) {
console.log(err);
return next(`Error completing training job ${id}`);
}
res.json({ id });
});
app.get('/job/:id', async (req : Request, res : Response, next : NextFunction) => { app.get('/job/:id', async (req : Request, res : Response, next : NextFunction) => {
let jobStatus : string; let jobStatus : string;
@ -416,29 +344,16 @@ app.get('/dataset/:id', async (req : Request, res: Response, next : NextFunction
}); });
app.get('/job', async (req : Request, res: Response, next : NextFunction) => { app.get('/job', async (req : Request, res: Response, next : NextFunction) => {
let jobArr : string[]; let jobId : string;
try { try {
jobArr = await job(); jobId = await job();
} catch (err) { } catch (err) {
console.error(err); console.error(err);
return next('Error getting job'); return next('Error getting job');
} }
res.json(jobArr); res.json([jobId]);
});
app.get('/jobs', async (req : Request, res: Response, next : NextFunction) => {
let jobArr : string[];
try {
jobArr = await jobs();
} catch (err) {
console.error(err);
return next('Error getting job');
}
res.json(jobArr);
}); });
app.post('/job/claim/:id', async (req : Request, res: Response, next : NextFunction) => { app.post('/job/claim/:id', async (req : Request, res: Response, next : NextFunction) => {
@ -455,24 +370,22 @@ app.post('/job/claim/:id', async (req : Request, res: Response, next : NextFunct
try { try {
jobObj = await claim(id); jobObj = await claim(id);
console.log(`Job ${id} was claimed`);
} catch (err) { } catch (err) {
console.error(err); console.error(err);
return next('Error claiming job'); return next('Error claiming job');
} }
resObj.id = id; resJob.id = id;
resObj.path = `/dataset/${id}`; resJob.datasetPath = `/dataset/${id}`;
resObj.dataset = jobObj.dataset; resJob.model = jobObj.model;
resObj.model = jobObj.model;
resObj.name = jobObj.name;
res.json(resObj); res.json(resJob);
}); });
app.post('/job/fail/:id', async (req : Request, res: Response, next : NextFunction) => { app.post('/job/fail/:id', async (req : Request, res: Response, next : NextFunction) => {
let id : string; let id : string;
let meta : string = null; let jobObj : any;
let resObj : any = {};
if (typeof req.params.id === 'undefined' || req.params.id === null) { if (typeof req.params.id === 'undefined' || req.params.id === null) {
console.error(`No dataset id provided`); console.error(`No dataset id provided`);
@ -480,20 +393,22 @@ app.post('/job/fail/:id', async (req : Request, res: Response, next : NextFuncti
} }
id = req.params.id; id = req.params.id;
if (typeof req.body.meta !== 'undefined') {
meta = req.body.meta;
}
try { try {
await fail(id, meta); jobObj = await claim(id);
console.log(`Job ${id} failed`);
} catch (err) { } catch (err) {
console.error(err); console.error(err);
return next('Error failing job'); return next('Error claiming job');
} }
res.json(true); resJob.id = id;
resJob.datasetPath = `/dataset/${id}`;
resJob.model = jobObj.model;
res.json(resJob);
}); });
//app.get('/jobs');
//app.post('/job/started/:id', )
app.listen(port, () => { app.listen(port, () => {
console.log(`yolo_web running on port ${port}`); console.log(`yolo_web running on port ${port}`);