mattmcw
/
photosite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Combine parsed metadata object into single photo record. Maybe move dimensions code out of the for loop into that async

This commit is contained in:
mmcwilliams 2024-12-05 14:55:12 -05:00
parent 175dca1670
commit 606df7a9dc
11 changed files with 157 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
README.md
View File

@ -15,7 +15,7 @@ Create a `.env` file by copying the `default.env` file and modifying the values
The following command will sync the data with s3, generate metadata for each new photo and produce all required thumbnails required for the site.
```bash
npm run generate
bash scripts/generate.sh
```
### Build Site
@ -23,5 +23,26 @@ npm run generate
This command will build the website using the sqlite + photo data available at the time it is run.
```bash
npm run build
bash scripts/build.sh
```
### File Naming
Photos should be named with relavent metadata in the filename so that the scripts can parse out the data.
The underscore character (`_`) is the delimiter between elements and dash (`-`) should be used in place of spaces.
The hash character (`#`) should be used to split between the metadata filename and the original file, so that it can be searched for if needed.
Elements are expected in the following order:
1. year
1. month
1. day
1. format
1. filmstock
1. location
1. description
1. original
```
2024_12_02_35mm_Kodak-Gold-200_Somerville-MA_Walk-towards-Harvard-Square#000061280009.tif
```

21
dist/db/index.js vendored
View File

@ -19,6 +19,27 @@ class DB {
});
});
}
//CASE WHEN LOWER(active) = 'true' THEN 1 ELSE 0 END AS active_bool
async create(photo) {
const keys = Object.keys(photo);
const query = `INSERT INTO photos (${keys.join(',')}) VALUES (${keys.map(el => '?').join(',')});`;
const values = [];
for (let key of keys) {
if (typeof photo[key] === 'boolean') {
values.push(photo[key] ? 1 : 0);
}
else {
values.push(photo[key]);
}
}
try {
await this.run(query, values);
this.log.info(`Inserted new photo`);
}
catch (err) {
this.log.error(`Error inserting record into photos`, err);
}
}
}
exports.DB = DB;
module.exports = { DB };

2
dist/db/index.js.map vendored
View File

@ -1 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":";;;AAAA,yBAAuB;AAEvB,gCAAmC;AAEnC,qCAAmC;AACnC,gCAAmC;AAYnC,MAAa,EAAE;IAId;QACC,IAAI,CAAC,GAAG,GAAG,IAAA,eAAS,EAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,CAAC,EAAE,GAAG,IAAI,kBAAQ,CAAC,IAAA,eAAS,EAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;IACzD,CAAC;IAEO,KAAK,CAAC,GAAG,CAAE,KAAc,EAAE,OAAe,IAAI;QACrD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAkB,EAAE,MAAiB,EAAE,EAAE;YAC5D,OAAO,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;gBAC7D,IAAI,GAAG;oBAAE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC5B,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACJ,CAAC;CAED;AAlBD,gBAkBC;AAED,MAAM,CAAC,OAAO,GAAG,EAAE,EAAE,EAAE,CAAC"}
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":";;;AAAA,yBAAuB;AAEvB,gCAAmC;AAEnC,qCAAmC;AACnC,gCAAmC;AAmBnC,MAAa,EAAE;IAId;QACC,IAAI,CAAC,GAAG,GAAG,IAAA,eAAS,EAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,CAAC,EAAE,GAAG,IAAI,kBAAQ,CAAC,IAAA,eAAS,EAAC,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;IACzD,CAAC;IAEO,KAAK,CAAC,GAAG,CAAE,KAAc,EAAE,OAAe,IAAI;QACrD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAkB,EAAE,MAAiB,EAAE,EAAE;YAC5D,OAAO,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,GAAW,EAAE,IAAY,EAAE,EAAE;gBAC7D,IAAI,GAAG;oBAAE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC5B,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;YACtB,CAAC,CAAC,CAAC;QACJ,CAAC,CAAC,CAAC;IACJ,CAAC;IAEF,mEAAmE;IAC3D,KAAK,CAAC,MAAM,CAAE,KAAa;QACjC,MAAM,IAAI,GAAc,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAY,uBAAuB,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,GAAG,CAAC,EAAE,CAAA,EAAE,CAAA,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;QACzG,MAAM,MAAM,GAAW,EAAE,CAAC;QAC1B,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YACtB,IAAI,OAAQ,KAAa,CAAC,GAAG,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC9C,MAAM,CAAC,IAAI,CAAE,KAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1C,CAAC;iBAAM,CAAC;gBACP,MAAM,CAAC,IAAI,CAAE,KAAa,CAAC,GAAG,CAAC,CAAC,CAAC;YAClC,CAAC;QACF,CAAC;QACD,IAAI,CAAC;YACJ,MAAM,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC9B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAA;QACpC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,oCAAoC,EAAE,GAAG,CAAC,CAAC;QAC3D,CAAC;IACF,CAAC;CAED;AAtCD,gBAsCC;AAED,MAAM,CAAC,OAAO,GAAG,EAAE,EAAE,EAAE,CAAC"}

20
dist/generate.js vendored
View File

@ -6,6 +6,7 @@ const promises_1 = require("fs/promises");
const path_1 = require("path");
const util_1 = require("util");
const shell_1 = require("./shell");
const hash_1 = require("./hash");
const files3_1 = require("./files3");
const env_1 = require("./env");
const db_1 = require("./db");
@ -32,6 +33,7 @@ class Generate {
let filename;
let meta;
let dimensions;
let photo;
try {
inbox = await (0, promises_1.realpath)(this.inbox);
}
@ -69,6 +71,7 @@ class Generate {
dimensions = await this.getImageDimensions(image);
meta.width = dimensions.width;
meta.height = dimensions.height;
photo = await this.createPhoto(image, meta);
console.dir(meta);
}
}
@ -110,7 +113,7 @@ class Generate {
//location
//description
//original
//2024_12_02_35mm_Kodak-Gold-200_Somerville-MA_Walk-with-Charlie#000061280009.tif
//2024_12_02_35mm_Kodak-Gold-200_Somerville-MA_Walk-towards-Harvard-Square#000061280009.tif
parseFilename(filename) {
const halves = filename.split('#');
const parts = halves[0].split('_');
@ -143,6 +146,21 @@ class Generate {
meta.original = halves[1];
return meta;
}
async createPhoto(image, meta) {
const hash = await hash_1.Hashes.fileHash(image);
return {
name: (0, path_1.basename)(image),
original: meta.original,
hash,
width: meta.width,
height: meta.height,
format: meta.format,
filmstock: meta.filmstock,
location: meta.location,
discovered: Date.now(),
created: +new Date(meta.year, meta.month, meta.day)
};
}
}
new Generate();
//# sourceMappingURL=generate.js.map

2
dist/generate.js.map vendored
View File

File diff suppressed because one or more lines are too long

29
dist/hash/index.js vendored
View File

@ -1,16 +1,23 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.hash = hash;
const fs_1 = require("fs");
exports.Hashes = void 0;
const crypto_1 = require("crypto");
function hash(path) {
return new Promise((resolve, reject) => {
const hashSum = (0, crypto_1.createHash)('sha256');
const stream = (0, fs_1.createReadStream)(path);
stream.on('error', (err) => reject(err));
stream.on('data', (chunk) => hashSum.update(chunk));
stream.on('end', () => resolve(hashSum.digest('hex')));
});
const fs_1 = require("fs");
class Hashes {
static async fileHash(path) {
return new Promise((resolve, reject) => {
const hashSum = (0, crypto_1.createHash)('sha256');
const stream = (0, fs_1.createReadStream)(path);
stream.on('error', (err) => reject(err));
stream.on('data', (chunk) => hashSum.update(chunk));
stream.on('end', () => resolve(hashSum.digest('hex')));
});
}
static stringHash(str) {
const sha = (0, crypto_1.createHash)('sha256').update(str);
return sha.digest('hex');
}
}
module.exports = { hash };
exports.Hashes = Hashes;
module.exports = { Hashes };
//# sourceMappingURL=index.js.map

2
dist/hash/index.js.map vendored
View File

@ -1 +1 @@
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/hash/index.ts"],"names":[],"mappings":";;AAGA,oBAQC;AAXD,2BAAsC;AACtC,mCAA0C;AAE1C,SAAgB,IAAI,CAAE,IAAa;IAClC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAkB,EAAE,MAAiB,EAAE,EAAE;QAC5D,MAAM,OAAO,GAAU,IAAA,mBAAU,EAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAS,IAAA,qBAAgB,EAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAc,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC7D,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACxD,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,OAAO,GAAG,EAAE,IAAI,EAAE,CAAC"}
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/hash/index.ts"],"names":[],"mappings":";;;AAAA,mCAA0C;AAC1C,2BAAsC;AAEtC,MAAa,MAAM;IAClB,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAE,IAAa;QACnC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAkB,EAAE,MAAiB,EAAE,EAAE;YAC5D,MAAM,OAAO,GAAU,IAAA,mBAAU,EAAC,QAAQ,CAAC,CAAC;YAC5C,MAAM,MAAM,GAAS,IAAA,qBAAgB,EAAC,IAAI,CAAC,CAAC;YAC5C,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAc,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC7D,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,UAAU,CAAE,GAAY;QAC9B,MAAM,GAAG,GAAU,IAAA,mBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACpD,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC;CACD;AAfD,wBAeC;AAED,MAAM,CAAC,OAAO,GAAG,EAAE,MAAM,EAAE,CAAC"}

3
sql/setup.sql
View File

@ -5,10 +5,11 @@ CREATE TABLE IF NOT EXISTS photos (
width INTEGER,
height INTEGER,
filmstock TEXT,
location TEXT,
discovered INTEGER,
created INTEGER,
updated INTEGER,
posted INTEGER DEFAULT 0,
bsky INTEGER DEFAULT 0,
score INTEGER DEFAULT 0,
deleted INTEGER DEFAULT 0
);

35
src/db/index.ts
View File

@ -5,13 +5,20 @@ import type { Logger } from 'winston';
import { Database } from 'sqlite3';
import { envString } from '../env';
interface Photos {
interface Photo {
name : string;
original? : string;
hash : string;
width : number;
height : number;
discovered ?: number;
posted? : boolean;
format? : string;
filmstock? : string;
location? : string;
discovered ? : number;
created? : number;
updated? : number;
bsky? : boolean;
deleted? : boolean;
score? : number;
}
@ -33,7 +40,27 @@ export class DB {
});
}
//CASE WHEN LOWER(active) = 'true' THEN 1 ELSE 0 END AS active_bool
public async create (photo : Photo) {
const keys : string[] = Object.keys(photo);
const query : string = `INSERT INTO photos (${keys.join(',')}) VALUES (${keys.map(el=>'?').join(',')});`;
const values : any[] = [];
for (let key of keys) {
if (typeof (photo as any)[key] === 'boolean') {
values.push((photo as any)[key] ? 1 : 0);
} else {
values.push((photo as any)[key]);
}
}
try {
await this.run(query, values);
this.log.info(`Inserted new photo`)
} catch (err) {
this.log.error(`Error inserting record into photos`, err);
}
}
}
module.exports = { DB };
export type { Photos };
export type { Photo };

25
src/generate.ts
View File

@ -5,10 +5,11 @@ import { readFile, readdir, realpath } from 'fs/promises';
import { join, basename } from 'path';
import { promisify } from 'util';
import { Shell } from './shell';
import { hash } from './hash';
import { Hashes } from './hash';
import { Files3 } from './files3'
import { envString } from './env';
import { DB } from './db';
import type { Photo } from './db';
const sizeOf = promisify(require('image-size'));
@ -55,6 +56,7 @@ class Generate {
let filename : string;
let meta : Metadata;
let dimensions : any;
let photo : Photo;
try {
inbox = await realpath(this.inbox);
@ -96,6 +98,9 @@ class Generate {
dimensions = await this.getImageDimensions(image);
meta.width = dimensions.width;
meta.height = dimensions.height;
photo = await this.createPhoto(image, meta);
console.dir(meta)
}
}
@ -141,7 +146,7 @@ class Generate {
//description
//original
//2024_12_02_35mm_Kodak-Gold-200_Somerville-MA_Walk-with-Charlie#000061280009.tif
//2024_12_02_35mm_Kodak-Gold-200_Somerville-MA_Walk-towards-Harvard-Square#000061280009.tif
private parseFilename (filename : string) : Metadata {
const halves : string[] = filename.split('#')
@ -175,6 +180,22 @@ class Generate {
meta.original = halves[1];
return meta;
}
private async createPhoto (image : string, meta : Metadata) : Promise<Photo> {
const hash : string = await Hashes.fileHash(image);
return {
name : basename(image),
original: meta.original,
hash,
width : meta.width,
height : meta.height,
format : meta.format,
filmstock : meta.filmstock,
location : meta.location,
discovered : Date.now(),
created : + new Date(meta.year, meta.month, meta.day)
}
}
}
new Generate();

27
src/hash/index.ts
View File

@ -1,14 +1,21 @@
import { createReadStream } from 'fs';
import { createHash, Hash } from 'crypto';
import { createReadStream } from 'fs';
export function hash (path : string) : Promise<string> {
return new Promise((resolve : Function, reject : Function) => {
const hashSum : Hash = createHash('sha256');
const stream : any = createReadStream(path);
stream.on('error', (err : Error) => reject(err));
stream.on('data', (chunk : Buffer) => hashSum.update(chunk));
stream.on('end', () => resolve(hashSum.digest('hex')));
});
export class Hashes {
static async fileHash (path : string) : Promise<string> {
return new Promise((resolve : Function, reject : Function) => {
const hashSum : Hash = createHash('sha256');
const stream : any = createReadStream(path);
stream.on('error', (err : Error) => reject(err));
stream.on('data', (chunk : Buffer) => hashSum.update(chunk));
stream.on('end', () => resolve(hashSum.digest('hex')));
});
}
static stringHash (str : string) : string {
const sha : Hash = createHash('sha256').update(str);
return sha.digest('hex');
}
}
module.exports = { hash };
module.exports = { Hashes };