Implemented client in node. Still relies on train.sh. Will consider rewriting it but it works for the moment.
This commit is contained in:
parent
8ff6b4ffdb
commit
7f92938a43
|
|
@ -0,0 +1,174 @@
|
|||
"use strict";
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const node_fetch_1 = __importDefault(require("node-fetch"));
|
||||
const log_1 = require("./log");
|
||||
const dotenv_1 = __importDefault(require("dotenv"));
|
||||
const child_process_1 = require("child_process");
|
||||
dotenv_1.default.config();
|
||||
const log = (0, log_1.createLog)('client');
|
||||
class Client {
|
||||
constructor() {
|
||||
this.maxJobs = 1;
|
||||
this.url = 'http://localhost:3333';
|
||||
this.jobs = [];
|
||||
if (typeof process.env.YOLO_WEB_URL !== 'undefined') {
|
||||
this.url = process.env.YOLO_WEB_URL;
|
||||
}
|
||||
this.cron();
|
||||
}
|
||||
cron() {
|
||||
this.job();
|
||||
setInterval(this.job.bind(this), 60 * 1000);
|
||||
}
|
||||
job() {
|
||||
if (this.jobs.length >= this.maxJobs) {
|
||||
return;
|
||||
}
|
||||
this.poll();
|
||||
}
|
||||
async poll() {
|
||||
const options = {
|
||||
method: 'GET'
|
||||
};
|
||||
const url = `${this.url}/job`;
|
||||
let res;
|
||||
let json;
|
||||
try {
|
||||
log.info('Checking for new jobs...');
|
||||
res = await (0, node_fetch_1.default)(url, options);
|
||||
}
|
||||
catch (err) {
|
||||
log.error(`ERROR getting ${url}`);
|
||||
log.error(err);
|
||||
process.exit(1);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
json = await res.json();
|
||||
}
|
||||
catch (err) {
|
||||
log.error('ERROR parsing JSON');
|
||||
log.error(err);
|
||||
process.exit(2);
|
||||
return;
|
||||
}
|
||||
if (json === null || json.length === 0) {
|
||||
log.info('No jobs found.');
|
||||
return;
|
||||
}
|
||||
await this.claim(json[0]);
|
||||
}
|
||||
async claim(id) {
|
||||
const options = {
|
||||
method: 'POST'
|
||||
};
|
||||
const url = `${this.url}/job/claim/${id}`;
|
||||
let res;
|
||||
let json;
|
||||
try {
|
||||
log.info(`Claiming job ${id}...`);
|
||||
res = await (0, node_fetch_1.default)(url, options);
|
||||
}
|
||||
catch (err) {
|
||||
log.error(`ERROR posting to ${url}`);
|
||||
log.error(err);
|
||||
process.exit(1);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
json = await res.json();
|
||||
log.info(json);
|
||||
}
|
||||
catch (err) {
|
||||
log.error('ERROR parsing JSON');
|
||||
log.error(err);
|
||||
process.exit(2);
|
||||
return;
|
||||
}
|
||||
if (this.jobs.indexOf(id) === -1) {
|
||||
this.jobs.push(id);
|
||||
log.info(`Claimed job ${id}`);
|
||||
this.train(json);
|
||||
}
|
||||
}
|
||||
async train(job) {
|
||||
const cmd = 'bash';
|
||||
const args = ['./scripts/train.sh', JSON.stringify(job)];
|
||||
const child = (0, child_process_1.spawn)(cmd, args);
|
||||
child.stdout.on('data', (data) => {
|
||||
const line = data.toString().trim();
|
||||
if (line === '')
|
||||
return;
|
||||
log.info(line);
|
||||
});
|
||||
child.stderr.on('data', (data) => {
|
||||
const line = data.toString().trim();
|
||||
if (line === '')
|
||||
return;
|
||||
log.info(line);
|
||||
});
|
||||
/*child.on('close', (code : number) => {
|
||||
log.info(`child process close all stdio with code ${code}`);
|
||||
});*/
|
||||
child.on('exit', (code) => {
|
||||
log.info(`Train job completed with code ${code}`);
|
||||
if (code !== 0) {
|
||||
this.fail(job.id, `Train script exited with code ${code}`);
|
||||
}
|
||||
});
|
||||
child.on('error', (err) => {
|
||||
log.error('train job errored out');
|
||||
log.error(err);
|
||||
this.fail(job.id, err.toString());
|
||||
});
|
||||
}
|
||||
async fail(id, reason) {
|
||||
const meta = {
|
||||
meta: reason
|
||||
};
|
||||
const options = {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify(meta)
|
||||
};
|
||||
const url = `${this.url}/job/fail/${id}`;
|
||||
let res;
|
||||
let json;
|
||||
this.removeJob(id);
|
||||
try {
|
||||
log.info(`Failed job ${id}...`);
|
||||
res = await (0, node_fetch_1.default)(url, options);
|
||||
}
|
||||
catch (err) {
|
||||
log.error(`ERROR posting to ${url}`);
|
||||
log.error(err);
|
||||
process.exit(1);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
json = await res.json();
|
||||
log.info(json);
|
||||
}
|
||||
catch (err) {
|
||||
log.error('ERROR parsing JSON');
|
||||
log.error(err);
|
||||
process.exit(2);
|
||||
return;
|
||||
}
|
||||
log.info(json);
|
||||
}
|
||||
removeJob(id) {
|
||||
const index = this.jobs.indexOf(id);
|
||||
if (index !== -1) {
|
||||
this.jobs.splice(index, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
new Client();
|
||||
//# sourceMappingURL=client.js.map
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":";;;;;AAAA,4DAGmB;AACnB,+BAAkC;AAElC,oDAA4B;AAE5B,iDAAsC;AAGtC,gBAAM,CAAC,MAAM,EAAE,CAAC;AAEhB,MAAM,GAAG,GAAY,IAAA,eAAS,EAAC,QAAQ,CAAC,CAAC;AAEzC,MAAM,MAAM;IAKX;QAJQ,YAAO,GAAY,CAAC,CAAC;QACrB,QAAG,GAAY,uBAAuB,CAAC;QACvC,SAAI,GAAc,EAAE,CAAC;QAG5B,IAAI,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,WAAW,EAAE;YACpD,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;SACpC;QACD,IAAI,CAAC,IAAI,EAAE,CAAC;IACb,CAAC;IAEM,IAAI;QACV,IAAI,CAAC,GAAG,EAAE,CAAC;QACX,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC;IAC7C,CAAC;IAEO,GAAG;QACV,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE;YACrC,OAAO;SACP;QACD,IAAI,CAAC,IAAI,EAAE,CAAC;IACb,CAAC;IAEO,KAAK,CAAC,IAAI;QACjB,MAAM,OAAO,GAAS;YACrB,MAAM,EAAG,KAAK;SACd,CAAC;QACF,MAAM,GAAG,GAAY,GAAG,IAAI,CAAC,GAAG,MAAM,CAAC;QACvC,IAAI,GAAc,CAAC;QACnB,IAAI,IAAU,CAAC;QAEf,IAAI;YACH,GAAG,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;YACrC,GAAG,GAAG,MAAM,IAAA,oBAAK,EAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAChC;QAAC,OAAO,GAAG,EAAE;YACb,GAAG,CAAC,KAAK,CAAC,iBAAiB,GAAG,EAAE,CAAC,CAAC;YAClC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO;SACP;QAED,IAAI;YACH,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;SACxB;QAAC,OAAO,GAAG,EAAE;YACb,GAAG,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YAChC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO;SACP;QAED,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACvC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAC3B,OAAO;SACP;QAED,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,KAAK,CAAE,EAAW;QAC/B,MAAM,OAAO,GAAS;YACrB,MAAM,EAAG,MAAM;SACf,CAAC;QACF,MAAM,GAAG,GAAY,GAAG,IAAI,CAAC,GAAG,cAAc,EAAE,EAAE,CAAC;QACnD,IAAI,GAAc,CAAC;QACnB,IAAI,IAAU,CAAC;QAEf,IAAI;YACH,GAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;YAClC,GAAG,GAAG,MAAM,IAAA,oBAAK,EAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAChC;QAAC,OAAO,GAAG,EAAE;YACb,GAAG,CAAC,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;YACrC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO;SACP;QAED,IAAI;YACH,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACxB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACf;QAAC,OAAO,GAAG,EAAE;YACb,GAAG,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YAChC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO;SACP;QAED,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE;YACjC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;YAC9B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;SACjB;IACF,CAAC;IAEO,KAAK,CAAC,KAAK,CAAE,GAAS;QAC7B,MAAM,GAAG,GAAY,MAAM,CAAC;QAC5B,MAAM,IAAI,GAAc,CAAC,oBAAoB,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;QACpE,MAAM,KAAK,GAAkB,IAAA,qBAAK,EAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAE9C,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAa,EAAE,EAAE;YACzC,MAAM,IAAI,GAAY,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;YAC7C,IAAI,IAAI,KAAK,EAAE;gBAAE,OAAO;YACxB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAa,EAAE,EAAE;YACzC,MAAM,IAAI,GAAY,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;YAC7C,IAAI,IAAI,KAAK,EAAE;gBAAE,OAAO;YACxB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACb,CAAC,CAAC,CAAC;QAEN;;aAEK;QAEL,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAa,EAAE,EAAE;YAClC,GAAG,CAAC,IAAI,CAAC,iCAAiC,IAAI,EAAE,CAAC,CAAC;YAClD,IAAI,IAAI,KAAK,CAAC,EAAE;gBACf,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,iCAAiC,IAAI,EAAE,CAAC,CAAA;aAC1D;QACF,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAW,EAAE,EAAE;YACjC,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;YACnC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,IAAI,CAAE,EAAW,EAAE,MAAe;QAC/C,MAAM,IAAI,GAAS;YAClB,IAAI,EAAG,MAAM;SACb,CAAC;QACF,MAAM,OAAO,GAAS;YACrB,MAAM,EAAG,MAAM;YACf,OAAO,EAAE;gBACR,QAAQ,EAAE,kBAAkB;gBAC5B,cAAc,EAAE,kBAAkB;aAClC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC1B,CAAC;QACF,MAAM,GAAG,GAAY,GAAG,IAAI,CAAC,GAAG,aAAa,EAAE,EAAE,CAAC;QAClD,IAAI,GAAc,CAAC;QACnB,IAAI,IAAU,CAAC;QAEf,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAEnB,IAAI;YACH,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;YAChC,GAAG,GAAG,MAAM,IAAA,oBAAK,EAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAChC;QAAC,OAAO,GAAG,EAAE;YACb,GAAG,CAAC,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;YACrC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO;SACP;QAED,IAAI;YACH,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACxB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACf;QAAC,OAAO,GAAG,EAAE;YACb,GAAG,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YAChC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO;SACP;QAED,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAEO,SAAS,CAAE,EAAW;QAC7B,MAAM,KAAK,GAAY,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7C,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE;YACjB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;SAC3B;IACF,CAAC;CACD;AAED,IAAI,MAAM,EAAE,CAAA"}
|
||||
|
|
@ -21,7 +21,7 @@ const dbPath = (0, path_1.join)(data, 'queue.sqlite');
|
|||
const app = (0, express_1.default)();
|
||||
const tmp = (0, os_1.tmpdir)();
|
||||
const db = new sqlite3_1.Database(dbPath);
|
||||
let log;
|
||||
let log = (0, log_1.createLog)('server');
|
||||
const accepted = ['application/zip', 'application/x-zip-compressed'];
|
||||
const storage = multer_1.default.diskStorage({
|
||||
destination: function (req, file, cb) {
|
||||
|
|
@ -467,7 +467,6 @@ app.post('/job/fail/:id', async (req, res, next) => {
|
|||
res.json(true);
|
||||
});
|
||||
app.listen(port, () => {
|
||||
log = (0, log_1.createLog)('server');
|
||||
log.info(`yolo_web running on port ${port}`);
|
||||
});
|
||||
//# sourceMappingURL=index.js.map
|
||||
File diff suppressed because one or more lines are too long
|
|
@ -10,11 +10,12 @@
|
|||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"body-parser": "^1.20.2",
|
||||
"dotenv": "^16.3.1",
|
||||
"express": "^4.18.2",
|
||||
"lodash": "^4.17.21",
|
||||
"mime": "^3.0.0",
|
||||
"multer": "^1.4.5-lts.1",
|
||||
"node-fetch": "^3.3.2",
|
||||
"node-fetch": "^2.7.0",
|
||||
"sqlite3": "^5.1.6",
|
||||
"triple-beam": "^1.4.1",
|
||||
"uuid": "^9.0.0",
|
||||
|
|
@ -74,25 +75,6 @@
|
|||
"node-pre-gyp": "bin/node-pre-gyp"
|
||||
}
|
||||
},
|
||||
"node_modules/@mapbox/node-pre-gyp/node_modules/node-fetch": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
||||
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
|
||||
"dependencies": {
|
||||
"whatwg-url": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "4.x || >=6.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"encoding": "^0.1.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"encoding": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@npmcli/fs": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-1.1.1.tgz",
|
||||
|
|
@ -687,14 +669,6 @@
|
|||
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
|
||||
"integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="
|
||||
},
|
||||
"node_modules/data-uri-to-buffer": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
|
||||
"integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
|
||||
"engines": {
|
||||
"node": ">= 12"
|
||||
}
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "2.6.9",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
|
||||
|
|
@ -742,6 +716,17 @@
|
|||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/dotenv": {
|
||||
"version": "16.3.1",
|
||||
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz",
|
||||
"integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/motdotla/dotenv?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/ee-first": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
|
||||
|
|
@ -897,28 +882,6 @@
|
|||
"resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz",
|
||||
"integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw=="
|
||||
},
|
||||
"node_modules/fetch-blob": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
|
||||
"integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/jimmywarting"
|
||||
},
|
||||
{
|
||||
"type": "paypal",
|
||||
"url": "https://paypal.me/jimmywarting"
|
||||
}
|
||||
],
|
||||
"dependencies": {
|
||||
"node-domexception": "^1.0.0",
|
||||
"web-streams-polyfill": "^3.0.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^12.20 || >= 14.13"
|
||||
}
|
||||
},
|
||||
"node_modules/finalhandler": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz",
|
||||
|
|
@ -955,17 +918,6 @@
|
|||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/formdata-polyfill": {
|
||||
"version": "4.0.10",
|
||||
"resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
|
||||
"integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
|
||||
"dependencies": {
|
||||
"fetch-blob": "^3.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.20.0"
|
||||
}
|
||||
},
|
||||
"node_modules/forwarded": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
|
||||
|
|
@ -1609,39 +1561,23 @@
|
|||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz",
|
||||
"integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ=="
|
||||
},
|
||||
"node_modules/node-domexception": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
|
||||
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/jimmywarting"
|
||||
},
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://paypal.me/jimmywarting"
|
||||
}
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=10.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-fetch": {
|
||||
"version": "3.3.2",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
|
||||
"integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
||||
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
|
||||
"dependencies": {
|
||||
"data-uri-to-buffer": "^4.0.0",
|
||||
"fetch-blob": "^3.1.4",
|
||||
"formdata-polyfill": "^4.0.10"
|
||||
"whatwg-url": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^12.20.0 || ^14.13.1 || >=16.0.0"
|
||||
"node": "4.x || >=6.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/node-fetch"
|
||||
"peerDependencies": {
|
||||
"encoding": "^0.1.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"encoding": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/node-gyp": {
|
||||
|
|
@ -2348,14 +2284,6 @@
|
|||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/web-streams-polyfill": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
|
||||
"integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
|
||||
"engines": {
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||
|
|
|
|||
|
|
@ -22,11 +22,12 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"body-parser": "^1.20.2",
|
||||
"dotenv": "^16.3.1",
|
||||
"express": "^4.18.2",
|
||||
"lodash": "^4.17.21",
|
||||
"mime": "^3.0.0",
|
||||
"multer": "^1.4.5-lts.1",
|
||||
"node-fetch": "^3.3.2",
|
||||
"node-fetch": "^2.7.0",
|
||||
"sqlite3": "^5.1.6",
|
||||
"triple-beam": "^1.4.1",
|
||||
"uuid": "^9.0.0",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,194 @@
|
|||
import fetch, {
|
||||
Request,
|
||||
Response,
|
||||
} from 'node-fetch'
|
||||
import { createLog } from './log';
|
||||
import type { Logger } from 'winston';
|
||||
import dotenv from 'dotenv';
|
||||
import fs from 'fs/promises';
|
||||
import { spawn } from 'child_process';
|
||||
import type { ChildProcess } from 'child_process';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
const log : Logger = createLog('client');
|
||||
|
||||
class Client {
|
||||
private maxJobs : number = 1;
|
||||
private url : string = 'http://localhost:3333';
|
||||
private jobs : string[] = [];
|
||||
|
||||
constructor () {
|
||||
if (typeof process.env.YOLO_WEB_URL !== 'undefined') {
|
||||
this.url = process.env.YOLO_WEB_URL;
|
||||
}
|
||||
this.cron();
|
||||
}
|
||||
|
||||
public cron () {
|
||||
this.job();
|
||||
setInterval(this.job.bind(this), 60 * 1000);
|
||||
}
|
||||
|
||||
private job () {
|
||||
if (this.jobs.length >= this.maxJobs) {
|
||||
return;
|
||||
}
|
||||
this.poll();
|
||||
}
|
||||
|
||||
private async poll () {
|
||||
const options : any = {
|
||||
method : 'GET'
|
||||
};
|
||||
const url : string = `${this.url}/job`;
|
||||
let res : Response;
|
||||
let json : any;
|
||||
|
||||
try {
|
||||
log.info('Checking for new jobs...');
|
||||
res = await fetch(url, options);
|
||||
} catch (err) {
|
||||
log.error(`ERROR getting ${url}`);
|
||||
log.error(err);
|
||||
process.exit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
json = await res.json();
|
||||
} catch (err) {
|
||||
log.error('ERROR parsing JSON');
|
||||
log.error(err);
|
||||
process.exit(2);
|
||||
return;
|
||||
}
|
||||
|
||||
if (json === null || json.length === 0) {
|
||||
log.info('No jobs found.');
|
||||
return;
|
||||
}
|
||||
|
||||
await this.claim(json[0]);
|
||||
}
|
||||
|
||||
private async claim (id : string) {
|
||||
const options : any = {
|
||||
method : 'POST'
|
||||
};
|
||||
const url : string = `${this.url}/job/claim/${id}`;
|
||||
let res : Response;
|
||||
let json : any;
|
||||
|
||||
try {
|
||||
log.info(`Claiming job ${id}...`);
|
||||
res = await fetch(url, options);
|
||||
} catch (err) {
|
||||
log.error(`ERROR posting to ${url}`);
|
||||
log.error(err);
|
||||
process.exit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
json = await res.json();
|
||||
log.info(json);
|
||||
} catch (err) {
|
||||
log.error('ERROR parsing JSON');
|
||||
log.error(err);
|
||||
process.exit(2);
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.jobs.indexOf(id) === -1) {
|
||||
this.jobs.push(id);
|
||||
log.info(`Claimed job ${id}`);
|
||||
this.train(json);
|
||||
}
|
||||
}
|
||||
|
||||
private async train (job : any) {
|
||||
const cmd : string = 'bash';
|
||||
const args : string[] = ['./scripts/train.sh', JSON.stringify(job)];
|
||||
const child : ChildProcess = spawn(cmd, args);
|
||||
|
||||
child.stdout.on('data', (data : Buffer) => {
|
||||
const line : string = data.toString().trim();
|
||||
if (line === '') return;
|
||||
log.info(line);
|
||||
});
|
||||
|
||||
child.stderr.on('data', (data : Buffer) => {
|
||||
const line : string = data.toString().trim();
|
||||
if (line === '') return;
|
||||
log.info(line);
|
||||
});
|
||||
|
||||
/*child.on('close', (code : number) => {
|
||||
log.info(`child process close all stdio with code ${code}`);
|
||||
});*/
|
||||
|
||||
child.on('exit', (code : number) => {
|
||||
log.info(`Train job completed with code ${code}`);
|
||||
if (code !== 0) {
|
||||
this.fail(job.id, `Train script exited with code ${code}`)
|
||||
}
|
||||
});
|
||||
|
||||
child.on('error', (err : Error) => {
|
||||
log.error('train job errored out');
|
||||
log.error(err);
|
||||
this.fail(job.id, err.toString());
|
||||
});
|
||||
}
|
||||
|
||||
private async fail (id : string, reason : string) {
|
||||
const meta : any = {
|
||||
meta : reason
|
||||
};
|
||||
const options : any = {
|
||||
method : 'POST',
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify(meta)
|
||||
};
|
||||
const url : string = `${this.url}/job/fail/${id}`;
|
||||
let res : Response;
|
||||
let json : any;
|
||||
|
||||
this.removeJob(id);
|
||||
|
||||
try {
|
||||
log.info(`Failed job ${id}...`);
|
||||
res = await fetch(url, options);
|
||||
} catch (err) {
|
||||
log.error(`ERROR posting to ${url}`);
|
||||
log.error(err);
|
||||
process.exit(1);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
json = await res.json();
|
||||
log.info(json);
|
||||
} catch (err) {
|
||||
log.error('ERROR parsing JSON');
|
||||
log.error(err);
|
||||
process.exit(2);
|
||||
return;
|
||||
}
|
||||
|
||||
log.info(json);
|
||||
}
|
||||
|
||||
private removeJob (id : string) {
|
||||
const index : number = this.jobs.indexOf(id);
|
||||
if (index !== -1) {
|
||||
this.jobs.splice(index, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
new Client()
|
||||
Loading…
Reference in New Issue