Implemented client in node. Still relies on train.sh. Will consider rewriting it but it works for the moment.

dist/client.js

@@ -0,0 +1,174 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const node_fetch_1 = __importDefault(require("node-fetch"));
+const log_1 = require("./log");
+const dotenv_1 = __importDefault(require("dotenv"));
+const child_process_1 = require("child_process");
+dotenv_1.default.config();
+const log = (0, log_1.createLog)('client');
+class Client {
+    constructor() {
+        this.maxJobs = 1;
+        this.url = 'http://localhost:3333';
+        this.jobs = [];
+        if (typeof process.env.YOLO_WEB_URL !== 'undefined') {
+            this.url = process.env.YOLO_WEB_URL;
+        }
+        this.cron();
+    }
+    cron() {
+        this.job();
+        setInterval(this.job.bind(this), 60 * 1000);
+    }
+    job() {
+        if (this.jobs.length >= this.maxJobs) {
+            return;
+        }
+        this.poll();
+    }
+    async poll() {
+        const options = {
+            method: 'GET'
+        };
+        const url = `${this.url}/job`;
+        let res;
+        let json;
+        try {
+            log.info('Checking for new jobs...');
+            res = await (0, node_fetch_1.default)(url, options);
+        }
+        catch (err) {
+            log.error(`ERROR getting ${url}`);
+            log.error(err);
+            process.exit(1);
+            return;
+        }
+        try {
+            json = await res.json();
+        }
+        catch (err) {
+            log.error('ERROR parsing JSON');
+            log.error(err);
+            process.exit(2);
+            return;
+        }
+        if (json === null || json.length === 0) {
+            log.info('No jobs found.');
+            return;
+        }
+        await this.claim(json[0]);
+    }
+    async claim(id) {
+        const options = {
+            method: 'POST'
+        };
+        const url = `${this.url}/job/claim/${id}`;
+        let res;
+        let json;
+        try {
+            log.info(`Claiming job ${id}...`);
+            res = await (0, node_fetch_1.default)(url, options);
+        }
+        catch (err) {
+            log.error(`ERROR posting to ${url}`);
+            log.error(err);
+            process.exit(1);
+            return;
+        }
+        try {
+            json = await res.json();
+            log.info(json);
+        }
+        catch (err) {
+            log.error('ERROR parsing JSON');
+            log.error(err);
+            process.exit(2);
+            return;
+        }
+        if (this.jobs.indexOf(id) === -1) {
+            this.jobs.push(id);
+            log.info(`Claimed job ${id}`);
+            this.train(json);
+        }
+    }
+    async train(job) {
+        const cmd = 'bash';
+        const args = ['./scripts/train.sh', JSON.stringify(job)];
+        const child = (0, child_process_1.spawn)(cmd, args);
+        child.stdout.on('data', (data) => {
+            const line = data.toString().trim();
+            if (line === '')
+                return;
+            log.info(line);
+        });
+        child.stderr.on('data', (data) => {
+            const line = data.toString().trim();
+            if (line === '')
+                return;
+            log.info(line);
+        });
+        /*child.on('close', (code : number) => {
+            log.info(`child process close all stdio with code ${code}`);
+        });*/
+        child.on('exit', (code) => {
+            log.info(`Train job completed with code ${code}`);
+            if (code !== 0) {
+                this.fail(job.id, `Train script exited with code ${code}`);
+            }
+        });
+        child.on('error', (err) => {
+            log.error('train job errored out');
+            log.error(err);
+            this.fail(job.id, err.toString());
+        });
+    }
+    async fail(id, reason) {
+        const meta = {
+            meta: reason
+        };
+        const options = {
+            method: 'POST',
+            headers: {
+                'Accept': 'application/json',
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify(meta)
+        };
+        const url = `${this.url}/job/fail/${id}`;
+        let res;
+        let json;
+        this.removeJob(id);
+        try {
+            log.info(`Failed job ${id}...`);
+            res = await (0, node_fetch_1.default)(url, options);
+        }
+        catch (err) {
+            log.error(`ERROR posting to ${url}`);
+            log.error(err);
+            process.exit(1);
+            return;
+        }
+        try {
+            json = await res.json();
+            log.info(json);
+        }
+        catch (err) {
+            log.error('ERROR parsing JSON');
+            log.error(err);
+            process.exit(2);
+            return;
+        }
+        log.info(json);
+    }
+    removeJob(id) {
+        const index = this.jobs.indexOf(id);
+        if (index !== -1) {
+            this.jobs.splice(index, 1);
+        }
+    }
+}
+new Client();
+//# sourceMappingURL=client.js.map

dist/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":";;;;;AAAA,4DAGmB;AACnB,+BAAkC;AAElC,oDAA4B;AAE5B,iDAAsC;AAGtC,gBAAM,CAAC,MAAM,EAAE,CAAC;AAEhB,MAAM,GAAG,GAAY,IAAA,eAAS,EAAC,QAAQ,CAAC,CAAC;AAEzC,MAAM,MAAM;IAKX;QAJQ,YAAO,GAAY,CAAC,CAAC;QACrB,QAAG,GAAY,uBAAuB,CAAC;QACvC,SAAI,GAAc,EAAE,CAAC;QAG5B,IAAI,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,KAAK,WAAW,EAAE;YACpD,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;SACpC;QACD,IAAI,CAAC,IAAI,EAAE,CAAC;IACb,CAAC;IAEM,IAAI;QACV,IAAI,CAAC,GAAG,EAAE,CAAC;QACX,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC;IAC7C,CAAC;IAEO,GAAG;QACV,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE;YACrC,OAAO;SACP;QACD,IAAI,CAAC,IAAI,EAAE,CAAC;IACb,CAAC;IAEO,KAAK,CAAC,IAAI;QACjB,MAAM,OAAO,GAAS;YACrB,MAAM,EAAG,KAAK;SACd,CAAC;QACF,MAAM,GAAG,GAAY,GAAG,IAAI,CAAC,GAAG,MAAM,CAAC;QACvC,IAAI,GAAc,CAAC;QACnB,IAAI,IAAU,CAAC;QAEf,IAAI;YACH,GAAG,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;YACrC,GAAG,GAAG,MAAM,IAAA,oBAAK,EAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAChC;QAAC,OAAO,GAAG,EAAE;YACb,GAAG,CAAC,KAAK,CAAC,iBAAiB,GAAG,EAAE,CAAC,CAAC;YAClC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO;SACP;QAED,IAAI;YACH,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;SACxB;QAAC,OAAO,GAAG,EAAE;YACb,GAAG,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YAChC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO;SACP;QAED,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACvC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAC3B,OAAO;SACP;QAED,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,KAAK,CAAE,EAAW;QAC/B,MAAM,OAAO,GAAS;YACrB,MAAM,EAAG,MAAM;SACf,CAAC;QACF,MAAM,GAAG,GAAY,GAAG,IAAI,CAAC,GAAG,cAAc,EAAE,EAAE,CAAC;QACnD,IAAI,GAAc,CAAC;QACnB,IAAI,IAAU,CAAC;QAEf,IAAI;YACH,GAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;YAClC,GAAG,GAAG,MAAM,IAAA,oBAAK,EAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAChC;QAAC,OAAO,GAAG,EAAE;YACb,GAAG,CAAC,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;YACrC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO;SACP;QAED,IAAI;YACH,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACxB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACf;QAAC,OAAO,GAAG,EAAE;YACb,GAAG,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YAChC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO;SACP;QAED,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE;YACjC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACnB,GAAG,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;YAC9B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;SACjB;IACF,CAAC;IAEO,KAAK,CAAC,KAAK,CAAE,GAAS;QAC7B,MAAM,GAAG,GAAY,MAAM,CAAC;QAC5B,MAAM,IAAI,GAAc,CAAC,oBAAoB,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;QACpE,MAAM,KAAK,GAAkB,IAAA,qBAAK,EAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAE9C,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAa,EAAE,EAAE;YACzC,MAAM,IAAI,GAAY,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;YAC7C,IAAI,IAAI,KAAK,EAAE;gBAAE,OAAO;YACxB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAa,EAAE,EAAE;YACzC,MAAM,IAAI,GAAY,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;YAC7C,IAAI,IAAI,KAAK,EAAE;gBAAE,OAAO;YACxB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACb,CAAC,CAAC,CAAC;QAEN;;aAEK;QAEL,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAa,EAAE,EAAE;YAClC,GAAG,CAAC,IAAI,CAAC,iCAAiC,IAAI,EAAE,CAAC,CAAC;YAClD,IAAI,IAAI,KAAK,CAAC,EAAE;gBACf,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,iCAAiC,IAAI,EAAE,CAAC,CAAA;aAC1D;QACF,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAW,EAAE,EAAE;YACjC,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;YACnC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,IAAI,CAAE,EAAW,EAAE,MAAe;QAC/C,MAAM,IAAI,GAAS;YAClB,IAAI,EAAG,MAAM;SACb,CAAC;QACF,MAAM,OAAO,GAAS;YACrB,MAAM,EAAG,MAAM;YACf,OAAO,EAAE;gBACR,QAAQ,EAAE,kBAAkB;gBAC5B,cAAc,EAAE,kBAAkB;aAClC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC1B,CAAC;QACF,MAAM,GAAG,GAAY,GAAG,IAAI,CAAC,GAAG,aAAa,EAAE,EAAE,CAAC;QAClD,IAAI,GAAc,CAAC;QACnB,IAAI,IAAU,CAAC;QAEf,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAEnB,IAAI;YACH,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;YAChC,GAAG,GAAG,MAAM,IAAA,oBAAK,EAAC,GAAG,EAAE,OAAO,CAAC,CAAC;SAChC;QAAC,OAAO,GAAG,EAAE;YACb,GAAG,CAAC,KAAK,CAAC,oBAAoB,GAAG,EAAE,CAAC,CAAC;YACrC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO;SACP;QAED,IAAI;YACH,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACxB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACf;QAAC,OAAO,GAAG,EAAE;YACb,GAAG,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YAChC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAChB,OAAO;SACP;QAED,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IAEO,SAAS,CAAE,EAAW;QAC7B,MAAM,KAAK,GAAY,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7C,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE;YACjB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;SAC3B;IACF,CAAC;CACD;AAED,IAAI,MAAM,EAAE,CAAA"}

dist/index.js

@@ -21,7 +21,7 @@ const dbPath = (0, path_1.join)(data, 'queue.sqlite');
 const app = (0, express_1.default)();
 const tmp = (0, os_1.tmpdir)();
 const db = new sqlite3_1.Database(dbPath);
-let log;
+let log = (0, log_1.createLog)('server');
 const accepted = ['application/zip', 'application/x-zip-compressed'];
 const storage = multer_1.default.diskStorage({
     destination: function (req, file, cb) {
@@ -467,7 +467,6 @@ app.post('/job/fail/:id', async (req, res, next) => {
     res.json(true);
 });
 app.listen(port, () => {
-    log = (0, log_1.createLog)('server');
     log.info(`yolo_web running on port ${port}`);
 });
 //# sourceMappingURL=index.js.map

package-lock.json

@@ -10,11 +10,12 @@
       "license": "ISC",
       "dependencies": {
         "body-parser": "^1.20.2",
+        "dotenv": "^16.3.1",
         "express": "^4.18.2",
         "lodash": "^4.17.21",
         "mime": "^3.0.0",
         "multer": "^1.4.5-lts.1",
-        "node-fetch": "^3.3.2",
+        "node-fetch": "^2.7.0",
         "sqlite3": "^5.1.6",
         "triple-beam": "^1.4.1",
         "uuid": "^9.0.0",
@@ -74,25 +75,6 @@
         "node-pre-gyp": "bin/node-pre-gyp"
       }
     },
-    "node_modules/@mapbox/node-pre-gyp/node_modules/node-fetch": {
-      "version": "2.7.0",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
-      "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
-      "dependencies": {
-        "whatwg-url": "^5.0.0"
-      },
-      "engines": {
-        "node": "4.x || >=6.0.0"
-      },
-      "peerDependencies": {
-        "encoding": "^0.1.0"
-      },
-      "peerDependenciesMeta": {
-        "encoding": {
-          "optional": true
-        }
-      }
-    },
     "node_modules/@npmcli/fs": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-1.1.1.tgz",
@@ -687,14 +669,6 @@
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
       "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ=="
     },
-    "node_modules/data-uri-to-buffer": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz",
-      "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==",
-      "engines": {
-        "node": ">= 12"
-      }
-    },
     "node_modules/debug": {
       "version": "2.6.9",
       "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -742,6 +716,17 @@
         "node": ">=8"
       }
     },
+    "node_modules/dotenv": {
+      "version": "16.3.1",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz",
+      "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://github.com/motdotla/dotenv?sponsor=1"
+      }
+    },
     "node_modules/ee-first": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
@@ -897,28 +882,6 @@
       "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz",
       "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw=="
     },
-    "node_modules/fetch-blob": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz",
-      "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/jimmywarting"
-        },
-        {
-          "type": "paypal",
-          "url": "https://paypal.me/jimmywarting"
-        }
-      ],
-      "dependencies": {
-        "node-domexception": "^1.0.0",
-        "web-streams-polyfill": "^3.0.3"
-      },
-      "engines": {
-        "node": "^12.20 || >= 14.13"
-      }
-    },
     "node_modules/finalhandler": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz",
@@ -955,17 +918,6 @@
         "node": ">= 6"
       }
     },
-    "node_modules/formdata-polyfill": {
-      "version": "4.0.10",
-      "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz",
-      "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==",
-      "dependencies": {
-        "fetch-blob": "^3.1.2"
-      },
-      "engines": {
-        "node": ">=12.20.0"
-      }
-    },
     "node_modules/forwarded": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@@ -1609,39 +1561,23 @@
       "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-4.3.0.tgz",
       "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ=="
     },
-    "node_modules/node-domexception": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
-      "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/jimmywarting"
-        },
-        {
-          "type": "github",
-          "url": "https://paypal.me/jimmywarting"
-        }
-      ],
-      "engines": {
-        "node": ">=10.5.0"
-      }
-    },
     "node_modules/node-fetch": {
-      "version": "3.3.2",
+      "version": "2.7.0",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
-      "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==",
+      "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
       "dependencies": {
-        "data-uri-to-buffer": "^4.0.0",
+        "whatwg-url": "^5.0.0"
-        "fetch-blob": "^3.1.4",
-        "formdata-polyfill": "^4.0.10"
       },
       "engines": {
-        "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+        "node": "4.x || >=6.0.0"
       },
-      "funding": {
+      "peerDependencies": {
-        "type": "opencollective",
+        "encoding": "^0.1.0"
-        "url": "https://opencollective.com/node-fetch"
+      },
+      "peerDependenciesMeta": {
+        "encoding": {
+          "optional": true
+        }
       }
     },
     "node_modules/node-gyp": {
@@ -2348,14 +2284,6 @@
         "node": ">= 0.8"
       }
     },
-    "node_modules/web-streams-polyfill": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz",
-      "integrity": "sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==",
-      "engines": {
-        "node": ">= 8"
-      }
-    },
     "node_modules/webidl-conversions": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",

package.json

@@ -22,11 +22,12 @@
   },
   "dependencies": {
     "body-parser": "^1.20.2",
+    "dotenv": "^16.3.1",
     "express": "^4.18.2",
     "lodash": "^4.17.21",
     "mime": "^3.0.0",
     "multer": "^1.4.5-lts.1",
-    "node-fetch": "^3.3.2",
+    "node-fetch": "^2.7.0",
     "sqlite3": "^5.1.6",
     "triple-beam": "^1.4.1",
     "uuid": "^9.0.0",

src/client.ts

@@ -0,0 +1,194 @@
+import fetch, {
+  Request,
+  Response,
+} from 'node-fetch'
+import { createLog } from './log';
+import type { Logger } from 'winston';
+import dotenv from 'dotenv';
+import fs from 'fs/promises';
+import { spawn } from 'child_process';
+import type { ChildProcess } from 'child_process';
+
+dotenv.config();
+
+const log : Logger = createLog('client');
+
+class Client {
+	private maxJobs : number = 1;
+	private url : string = 'http://localhost:3333';
+	private jobs : string[] = [];
+
+	constructor () {
+		if (typeof process.env.YOLO_WEB_URL !== 'undefined') {
+			this.url = process.env.YOLO_WEB_URL;
+		}
+		this.cron();
+	}
+
+	public cron () {
+		this.job();
+		setInterval(this.job.bind(this), 60 * 1000);
+	}
+
+	private job () {
+		if (this.jobs.length >= this.maxJobs) {
+			return;
+		}
+		this.poll();
+	}
+
+	private async poll () {
+		const options : any = {
+			method : 'GET'
+		};
+		const url : string = `${this.url}/job`;
+		let res : Response;
+		let json : any;
+
+		try {
+			log.info('Checking for new jobs...');
+			res = await fetch(url, options);
+		} catch (err) {
+			log.error(`ERROR getting ${url}`);
+			log.error(err);
+			process.exit(1);
+			return;
+		}
+		
+		try {
+			json = await res.json();
+		} catch (err) {
+			log.error('ERROR parsing JSON');
+			log.error(err);
+			process.exit(2);
+			return; 
+		}
+		
+		if (json === null || json.length === 0) {
+			log.info('No jobs found.');
+			return;
+		}
+
+		await this.claim(json[0]);
+	}
+
+	private async claim (id : string) {
+		const options : any = {
+			method : 'POST'
+		};
+		const url : string = `${this.url}/job/claim/${id}`;
+		let res : Response;
+		let json : any;
+
+		try {
+			log.info(`Claiming job ${id}...`);
+			res = await fetch(url, options);
+		} catch (err) {
+			log.error(`ERROR posting to ${url}`);
+			log.error(err);
+			process.exit(1);
+			return;
+		}
+		
+		try {
+			json = await res.json();
+			log.info(json);
+		} catch (err) {
+			log.error('ERROR parsing JSON');
+			log.error(err);
+			process.exit(2);
+			return; 
+		}
+
+		if (this.jobs.indexOf(id) === -1) {
+			this.jobs.push(id);
+			log.info(`Claimed job ${id}`);
+			this.train(json);
+		}
+	}
+
+	private async train (job : any) {
+		const cmd : string = 'bash';
+		const args : string[] = ['./scripts/train.sh', JSON.stringify(job)];
+		const child : ChildProcess = spawn(cmd, args);
+		
+		child.stdout.on('data', (data : Buffer) => {
+			const line : string = data.toString().trim();
+			if (line === '') return;
+			log.info(line);
+		});
+
+		child.stderr.on('data', (data : Buffer) => {
+			const line : string = data.toString().trim();
+			if (line === '') return;
+			log.info(line);
+    	});
+
+		/*child.on('close', (code : number) => {
+			log.info(`child process close all stdio with code ${code}`);
+		});*/
+
+		child.on('exit', (code : number) => {
+			log.info(`Train job completed with code ${code}`);
+			if (code !== 0) {
+				this.fail(job.id, `Train script exited with code ${code}`)
+			}
+		});
+
+		child.on('error', (err : Error) => {
+			log.error('train job errored out');
+			log.error(err);
+			this.fail(job.id, err.toString());
+		});
+	}
+
+	private async fail (id : string, reason : string) {
+		const meta : any = {
+			meta : reason
+		};
+		const options : any = {
+			method : 'POST',
+			headers: {
+				'Accept': 'application/json',
+				'Content-Type': 'application/json'
+			},
+			body: JSON.stringify(meta)
+		};
+		const url : string = `${this.url}/job/fail/${id}`;
+		let res : Response;
+		let json : any;
+
+		this.removeJob(id);
+
+		try {
+			log.info(`Failed job ${id}...`);
+			res = await fetch(url, options);
+		} catch (err) {
+			log.error(`ERROR posting to ${url}`);
+			log.error(err);
+			process.exit(1);
+			return;
+		}
+		
+		try {
+			json = await res.json();
+			log.info(json);
+		} catch (err) {
+			log.error('ERROR parsing JSON');
+			log.error(err);
+			process.exit(2);
+			return; 
+		}
+
+		log.info(json);
+	}
+
+	private removeJob (id : string) {
+		const index : number = this.jobs.indexOf(id);
+		if (index !== -1) {
+			this.jobs.splice(index, 1);
+		}
+	}
+}
+
+new Client()