mattmcw
/
photosite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

All work on generate project. Write to db. Start EXIF work

This commit is contained in:
mmcwilliams 2024-12-08 16:51:53 -05:00
parent 8915c9e1c6
commit 222a3c2ddc
14 changed files with 233 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
default.env
View File

@ -8,3 +8,4 @@ S3_BUCKET=""
S3_ENDPOINT=""
UMAMI=""
DB="data/site.db"
ARTIST="Unknown"

11
dist/db/index.js vendored
View File

@ -9,6 +9,7 @@ class DB {
constructor() {
this.log = (0, log_1.createLog)('db');
this.db = new sqlite3_1.Database((0, env_1.envString)('DB', 'data/site.db'));
this.db.run('PRAGMA journal_mode = WAL;');
}
async run(query, args = null) {
return new Promise((resolve, reject) => {
@ -19,6 +20,12 @@ class DB {
});
});
}
toBoolean(val) {
return val === 1 ? true : false;
}
fromBoolean(val) {
return val ? 1 : 0;
}
//CASE WHEN LOWER(active) = 'true' THEN 1 ELSE 0 END AS active_bool
async create(photo) {
const keys = Object.keys(photo);
@ -26,7 +33,7 @@ class DB {
const values = [];
for (let key of keys) {
if (typeof photo[key] === 'boolean') {
values.push(photo[key] ? 1 : 0);
values.push(this.fromBoolean(photo[key]));
}
else {
values.push(photo[key]);
@ -37,7 +44,7 @@ class DB {
this.log.info(`Inserted new photo`);
}
catch (err) {
this.log.error(`Error inserting record into photos`, err);
throw err;
}
}
}

2
dist/db/index.js.map vendored
View File

@ -1 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":";;;AAAA,yBAAuB;AAEvB,gCAAmC;AAEnC,qCAAmC;AACnC,gCAAmC;AAmBnC,MAAa,EAAE;IAId;QACC,IAAI,CAAC,GAAG,GAAG,IAAA,eAAS,EAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,CAAC,EAAE,GAAG,IAAI,kBAAQ,CAAC,IAAA,eAAS,EAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;IACzD,CAAC;IAEO,KAAK,CAAC,GAAG,CAAE,KAAc,EAAE,OAAe,IAAI;QACrD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAkB,EAAE,MAAiB,EAAE,EAAE;YAC5D,OAAO,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;gBAC7D,IAAI,GAAG;oBAAE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC5B,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACJ,CAAC;IAEF,mEAAmE;IAC3D,KAAK,CAAC,MAAM,CAAE,KAAa;QACjC,MAAM,IAAI,GAAc,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAY,uBAAuB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,GAAG,CAAC,EAAE,CAAA,EAAE,CAAA,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;QACzG,MAAM,MAAM,GAAW,EAAE,CAAC;QAC1B,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YACtB,IAAI,OAAQ,KAAa,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC9C,MAAM,CAAC,IAAI,CAAE,KAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACP,MAAM,CAAC,IAAI,CAAE,KAAa,CAAC,GAAG,CAAC,CAAC,CAAC;YAClC,CAAC;QACF,CAAC;QACD,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC9B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAA;QACpC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oCAAoC,EAAE,GAAG,CAAC,CAAC;QAC3D,CAAC;IACF,CAAC;CAED;AAtCD,gBAsCC;AAED,MAAM,CAAC,OAAO,GAAG,EAAE,EAAE,EAAE,CAAC"}
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":";;;AAAA,yBAAuB;AAEvB,gCAAmC;AAEnC,qCAAmC;AACnC,gCAAmC;AAmBnC,MAAa,EAAE;IAId;QACC,IAAI,CAAC,GAAG,GAAG,IAAA,eAAS,EAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,CAAC,EAAE,GAAG,IAAI,kBAAQ,CAAC,IAAA,eAAS,EAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;QACxD,IAAI,CAAC,EAAE,CAAC,GAAG,CAAE,4BAA4B,CAAC,CAAC;IAC5C,CAAC;IAEO,KAAK,CAAC,GAAG,CAAE,KAAc,EAAE,OAAe,IAAI;QACrD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAkB,EAAE,MAAiB,EAAE,EAAE;YAC5D,OAAO,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;gBAC7D,IAAI,GAAG;oBAAE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC5B,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACJ,CAAC;IAEO,SAAS,CAAE,GAAY;QAC9B,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;IACjC,CAAC;IAEO,WAAW,CAAE,GAAa;QACjC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,mEAAmE;IAC5D,KAAK,CAAC,MAAM,CAAE,KAAa;QACjC,MAAM,IAAI,GAAc,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAY,uBAAuB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,GAAG,CAAC,EAAE,CAAA,EAAE,CAAA,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;QACzG,MAAM,MAAM,GAAW,EAAE,CAAC;QAC1B,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YACtB,IAAI,OAAQ,KAAa,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC9C,MAAM,CAAC,IAAI,CAAE,IAAI,CAAC,WAAW,CAAG,KAAa,CAAC,GAAG,CAAC,CAAE,CAAC,CAAC;YACvD,CAAC;iBAAM,CAAC;gBACP,MAAM,CAAC,IAAI,CAAE,KAAa,CAAC,GAAG,CAAC,CAAC,CAAC;YAClC,CAAC;QACF,CAAC;QACD,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC9B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAA;QACpC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,GAAG,CAAC;QACX,CAAC;IACF,CAAC;CAED;AA/CD,gBA+CC;AAED,MAAM,CAAC,OAAO,GAAG,EAAE,EAAE,EAAE,CAAC"}

14
dist/files3/index.js vendored
View File

@ -107,7 +107,7 @@ class Files3 {
async create(file, keyName) {
if (!this.writeable)
return false;
const id = await this.hashFile(file);
const id = (0, uuid_1.v4)();
const ext = mime.extension(file.mimetype);
const key = typeof keyName !== 'undefined' ? keyName : `${id}.${ext}`;
const webPath = (0, path_1.join)('/files/', this.bucket, key);
@ -132,9 +132,9 @@ class Files3 {
record.hash = this.hash(file.buffer);
record.size = file.buffer.byteLength;
return new Promise((resolve, reject) => {
return this.s3.putObject(params, function (err, data) {
return this.s3.putObject(params, (err, data) => {
if (err) {
this.log.error(err);
this.log.error('create', err);
return reject(err);
}
else {
@ -321,6 +321,7 @@ class Files3 {
upload.concurrentParts(5);
return new Promise((resolve, reject) => {
upload.on('error', (err) => {
this.log.error('createStreamFromPath', err);
return reject(err);
});
upload.on('part', (details) => {
@ -351,6 +352,7 @@ class Files3 {
return new Promise((resolve, reject) => {
return this.s3.getObject(params, (err, data) => {
if (err) {
this.log.error('read', err);
return reject(err);
}
return resolve(data.Body); //buffer
@ -383,6 +385,7 @@ class Files3 {
return new Promise((resolve, reject) => {
return this.s3.getSignedUrl('putObject', s3Params, (err, url) => {
if (err) {
this.log.error('signedPutKey', err);
return reject(err);
}
return resolve(url);
@ -407,6 +410,7 @@ class Files3 {
return new Promise((resolve, reject) => {
return this.s3.getSignedUrl('getObject', s3Params, (err, url) => {
if (err) {
this.log.error('signedGetKey', err);
return reject(err);
}
return resolve(url);
@ -457,7 +461,7 @@ class Files3 {
return new Promise((resolve, reject) => {
return this.s3.deleteObject(params, (err, data) => {
if (err) {
this.log.error(err);
this.log.error('delete', err);
return reject(err);
}
return resolve(true); //buffer
@ -475,7 +479,7 @@ class Files3 {
return new Promise((resolve, reject) => {
return this.s3.listObjectsV2(params, (err, data) => {
if (err) {
this.log.error(err);
this.log.error('list', err);
return reject(err);
}
return resolve(data);

2
dist/files3/index.js.map vendored
View File

File diff suppressed because one or more lines are too long

30
dist/generate.js vendored
View File

@ -67,8 +67,28 @@ class Generate {
this.log.info(image);
filename = (0, path_1.basename)(image);
meta = this.parseFilename(filename);
try {
photo = await this.createPhoto(image, meta);
console.dir(meta);
}
catch (err) {
this.log.error(`Error creating photo record metadata`, err);
continue;
}
try {
await this.db.create(photo);
}
catch (err) {
this.log.error(`Error inserting photo into database`, err);
continue;
}
try {
await this.upload(image);
}
catch (err) {
this.log.error(`Error uploading image`, err);
continue;
}
//await this.move(image);
}
}
async img(file, exif) {
@ -145,6 +165,7 @@ class Generate {
async createPhoto(image, meta) {
const hash = await hash_1.Hashes.fileHash(image);
const dimensions = await this.getImageDimensions(image);
const now = Date.now();
return {
name: (0, path_1.basename)(image),
original: meta.original,
@ -154,10 +175,15 @@ class Generate {
format: meta.format,
filmstock: meta.filmstock,
location: meta.location,
discovered: Date.now(),
discovered: now,
updated: now,
created: +new Date(meta.year, meta.month, meta.day)
};
}
async upload(image) {
const name = (0, path_1.basename)(image);
return this.s3.createFromPath(image, name);
}
}
new Generate();
//# sourceMappingURL=generate.js.map

2
dist/generate.js.map vendored
View File

File diff suppressed because one or more lines are too long

8
notes/exit_test.txt Normal file
View File

@ -0,0 +1,8 @@
-XPTitle=Title here
-XPSubject=Subject here
-XPComment=Comments here
-XPArtist=Test Artist
-XPImageTitle
-XPImageUniqueID
-XPISOSpeed
-XPDateTimeOriginal

1
scripts/generate.sh
View File

@ -4,6 +4,7 @@ set -e
source .env
rm -rf data/site.db
mkdir -p data
cat "sql/setup.sql" | sqlite3 "${DB}"

8
scripts/img.sh
View File

@ -5,12 +5,12 @@ set -e
source .env
INPUT="${1}"
EXIF="${2}"
ID="${2}"
EXIF="${3}"
SIZES=(
"home:420"
"full:1920"
"bsky:2000"
"full:2000"
)
function img () {
@ -26,7 +26,7 @@ for sizeRaw in ${SIZES[@]}; do
size=$(echo $sizeRaw | awk -F':' '{print $2}')
name=$(basename "${1}")
name=${name%.*}
output="${WWW}/img/${name}_${size}.jpg"
output="${WWW}/img/${ID}_${size}.jpg"
img "${1}" "${output}" "${size}"
exiftool -overwrite_original -@ "${EXIF}" "${output}"
done

2
sql/setup.sql
View File

@ -1,9 +1,11 @@
CREATE TABLE IF NOT EXISTS photos (
id TEXT PRIMARY KEY,
name TEXT UNIQUE,
original TEXT UNIQUE,
hash TEXT UNIQUE,
width INTEGER,
height INTEGER,
format TEXT,
filmstock TEXT,
location TEXT,
discovered INTEGER,

55
src/db/index.ts
View File

@ -6,6 +6,7 @@ import { Database } from 'sqlite3';
import { envString } from '../env';
interface Photo {
id : string;
name : string;
original? : string;
hash : string;
@ -29,6 +30,7 @@ export class DB {
constructor () {
this.log = createLog('db');
this.db = new Database(envString('DB', 'data/site.db'));
this.db.run( 'PRAGMA journal_mode = WAL;');
}
private async run (query : string, args : any[] = null) {
@ -40,6 +42,23 @@ export class DB {
});
}
private async all (query : string, args : any[] = null) : Promise<any[]> {
return new Promise((resolve : Function, reject : Function) => {
return this.db.all(query, args, (err : Error, rows : any[]) => {
if (err) return reject(err);
return resolve(rows);
});
});
}
private toBoolean (val : number) : boolean {
return val === 1 ? true : false;
}
private fromBoolean (val : boolean) : number {
return val ? 1 : 0;
}
//CASE WHEN LOWER(active) = 'true' THEN 1 ELSE 0 END AS active_bool
public async create (photo : Photo) {
const keys : string[] = Object.keys(photo);
@ -47,19 +66,49 @@ export class DB {
const values : any[] = [];
for (let key of keys) {
if (typeof (photo as any)[key] === 'boolean') {
values.push((photo as any)[key] ? 1 : 0);
values.push( this.fromBoolean( (photo as any)[key] ));
} else {
values.push((photo as any)[key]);
}
}
try {
await this.run(query, values);
this.log.info(`Inserted new photo`)
this.log.info(`Inserted new photo ${photo.name}`);
} catch (err) {
this.log.error(`Error inserting record into photos`, err);
throw err;
}
}
public async existsName (name : string) : Promise<boolean> {
const query : string = `SELECT id FROM photos WHERE name = ? LIMIT 1;`;
let rows : any[] = [];
let exists : boolean = false;
try {
rows = await this.all(query, [ name ]);
} catch (err) {
this.log.error(`Error finding photo by name ${name}`, err);
}
if (rows.length > 0) {
exists = true;
}
return exists;
}
public async existsHash (hash : string) : Promise<boolean> {
const query : string = `SELECT id FROM photos WHERE hash = ? LIMIT 1;`;
let rows : any[] = [];
let exists : boolean = false;
try {
rows = await this.all(query, [ hash ]);
} catch (err) {
this.log.error(`Error finding photo by hash ${hash}`, err);
}
if (rows.length > 0) {
exists = true;
}
return exists;
}
}
module.exports = { DB };

21
src/files3/index.ts
View File

@ -51,6 +51,7 @@ export class Files3 {
endpoint: spacesEndpoint as unknown as string,
signatureVersion: 'v4'
};
this.endpoint = S3_ENDPOINT;
this.s3 = new S3(s3Config);
this.s3Stream = s3Stream(this.s3);
@ -97,11 +98,12 @@ export class Files3 {
**/
public async create (file : any, keyName? : string) {
if (!this.writeable) return false;
const id : string = await this.hashFile(file);
const id : string = uuid();
const ext : string | false = mime.extension(file.mimetype);
const key : string = typeof keyName !== 'undefined' ? keyName : `${id}.${ext}`;
const webPath : string = pathJoin('/files/', this.bucket, key);
const publicPath : string = pathJoin(`${this.bucket}.${this.endpoint}`, key);
const record : FileRecord = {
id,
created : +new Date(),
@ -113,6 +115,7 @@ export class Files3 {
type : file.mimetype,
size : null
};
const params : S3.PutObjectRequest = {
Bucket: this.bucket,
Key: key,
@ -124,10 +127,10 @@ export class Files3 {
record.size = file.buffer.byteLength;
return new Promise((resolve : Function, reject : Function) => {
return this.s3.putObject(params, function (err : Error, data : any) {
return this.s3.putObject(params, (err : Error, data : any) => {
if (err) {
this.log.error(err)
return reject(err)
this.log.error('create', err);
return reject(err);
} else {
this.log.info(`Saved file ${record.path}`);
return resolve(record);
@ -323,6 +326,7 @@ export class Files3 {
return new Promise((resolve : Function, reject : Function) => {
upload.on('error', (err : Error) => {
this.log.error('createStreamFromPath', err);
return reject(err);
});
upload.on('part', (details : any) => {
@ -355,6 +359,7 @@ export class Files3 {
return new Promise((resolve : Function, reject : Function) => {
return this.s3.getObject(params, (err : Error, data : any) => {
if (err) {
this.log.error('read', err);
return reject(err);
}
return resolve(data.Body) //buffer
@ -390,6 +395,7 @@ export class Files3 {
return new Promise((resolve : Function, reject : Function) => {
return this.s3.getSignedUrl('putObject', s3Params, (err : Error, url : string) => {
if (err) {
this.log.error('signedPutKey', err);
return reject(err);
}
return resolve(url);
@ -415,6 +421,7 @@ export class Files3 {
return new Promise((resolve : Function, reject : Function) => {
return this.s3.getSignedUrl('getObject', s3Params, (err : Error, url : string) => {
if (err) {
this.log.error('signedGetKey', err);
return reject(err);
}
return resolve(url);
@ -467,10 +474,10 @@ export class Files3 {
return new Promise((resolve : Function, reject : Function) => {
return this.s3.deleteObject(params, (err : Error, data : any) => {
if (err) {
this.log.error(err);
this.log.error('delete', err);
return reject(err);
}
return resolve(true) //buffer
return resolve(true); //buffer
});
});
}
@ -487,7 +494,7 @@ export class Files3 {
return new Promise((resolve : Function, reject : Function) => {
return this.s3.listObjectsV2(params, (err : Error, data : any) => {
if (err) {
this.log.error(err);
this.log.error('list', err);
return reject(err);
}
return resolve(data);

103
src/generate.ts
View File

@ -1,9 +1,12 @@
import 'dotenv/config';
import { createLog } from './log';
import type { Logger } from 'winston';
import { readFile, readdir, realpath } from 'fs/promises';
import { readFile, writeFile, readdir, realpath, rename } from 'fs/promises';
import { join, basename } from 'path';
import { promisify } from 'util';
import { v4 as uuid } from 'uuid';
import { randomBytes } from 'crypto';
import { tmpdir } from 'os';
import { Shell } from './shell';
import { Hashes } from './hash';
import { Files3 } from './files3'
@ -29,14 +32,16 @@ class Generate {
private files : string[];
private inbox : string = envString('INBOX', '~/Photos/toprocess');
private photos : string = envString('PHOTOS', '~/Photos/processed');
private artist : string = envString('ARTIST', 'Unknown');
private s3 : Files3;
private db : DB;
private tmp : string = tmpdir();
constructor () {
this.log = createLog('generate');
this.log.info(`Generating site: ${new Date()}`);
this.db = new DB();
this.s3 = new Files3(envString('S3_BUCKET', 'mmcwilliamsphotos'), true);
this.s3 = new Files3(envString('S3_BUCKET', 's3bucket'), true);
this.generate();
}
@ -90,15 +95,67 @@ class Generate {
);
for (let image of images) {
this.log.info(image);
filename = basename(image);
meta = this.parseFilename(filename);
try {
photo = await this.createPhoto(image, meta);
} catch (err) {
this.log.error(`Error creating photo record metadata`, err);
continue;
}
console.dir(meta)
if (await this.db.existsName(filename)) {
this.log.info(`Image ${filename} already exists`);
if (await this.db.existsHash(photo.hash)) {
this.log.warn(`Image ${name} already exists, moving...`);
await this.move(image);
continue;
}
}
if (await this.db.existsHash(photo.hash)) {
this.log.warn(`Image exists under different name, update?`);
await this.move(image);
continue;
}
try {
await this.db.create(photo);
} catch (err) {
this.log.error(`Error inserting photo into database`, err);
continue;
}
try {
await this.upload(image);
} catch (err) {
this.log.error(`Error uploading image`, err);
continue;
}
await this.move(image);
}
}
private async img (file : string, exif : string) {
//Artist
//ImageTitle
//ImageUniqueID
//ISOSpeed
//DateTimeOriginal
private async exif (photo : Photo) : Promise<string> {
const filePath : string = await this.mktemp('photosite_exif');
try {
} catch (err) {
}
return filePath;
}
private async img (file : string, id : string, exif : string) {
const cmd : string[] = ['bash', 'scripts/img.sh', file, exif];
const shell : Shell = new Shell(cmd);
try {
@ -177,7 +234,9 @@ class Generate {
private async createPhoto (image : string, meta : Metadata) : Promise<Photo> {
const hash : string = await Hashes.fileHash(image);
const dimensions : any = await this.getImageDimensions(image);
const now : number = Date.now();
return {
id : uuid(),
name : basename(image),
original: meta.original,
hash,
@ -186,10 +245,44 @@ class Generate {
format : meta.format,
filmstock : meta.filmstock,
location : meta.location,
discovered : Date.now(),
discovered : now,
updated : now,
created : + new Date(meta.year, meta.month, meta.day)
}
}
private async upload (image: string) {
const name : string = basename(image);
return this.s3.createFromPath(image, name);
}
private async move (image : string) {
const name : string = basename(image);
const dest : string = join(this.photos, name);
try {
await rename(image, dest);
this.log.info(`Moved image ${name} to outbox`);
} catch (err) {
this.log.error(`Error moving image`, err);
}
}
async mktemp (prefix : string = 'tmp') : Promise<string> {
const uniqueId = randomBytes(16).toString('hex');
const tempFilePath = join(this.tmp, `${prefix}-${uniqueId}`);
try {
await writeFile(tempFilePath, '', { flag: 'wx' });
return tempFilePath;
} catch (err) {
if (err.code === 'EEXIST') {
return this.mktemp(prefix);
} else {
throw err;
}
}
}
}
new Generate();