photosite/dist/files3/index.js

'use strict';
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.Files3 = void 0;
require("dotenv/config");
const uuid_1 = require("uuid");
const path_1 = require("path");
const promises_1 = require("fs/promises");
const fs_1 = require("fs");
const crypto_1 = require("crypto");
const mime = __importStar(require("mime-types"));
const aws_sdk_1 = require("aws-sdk");
const os_1 = require("os");
const s3Stream = require("s3-upload-stream");
const log_1 = require("../log");
const env_1 = require("../env");
const TMP_DIR = ((0, env_1.envString)('FILE3_DIR', null) !== null) ? (0, env_1.envString)('FILE3_DIR', '/tmp') : (0, env_1.envString)('TMP_DIR', (0, os_1.tmpdir)());
class Files3 {
    /**
     * @constructor
     *
     */
    constructor(bucket, writeable = true) {
        this.writeable = false;
        const S3_ENDPOINT = (0, env_1.envString)('S3_ENDPOINT', 'http://127.0.0.1:9000');
        const spacesEndpoint = new aws_sdk_1.Endpoint(S3_ENDPOINT);
        const s3Config = {
            accessKeyId: (0, env_1.envString)('S3_ACCESS_KEY', 'YOUR-ACCESSKEYID'),
            secretAccessKey: (0, env_1.envString)('S3_ACCESS_SECRET', 'YOUR-SECRETACCESSKEY'),
            endpoint: spacesEndpoint,
            signatureVersion: 'v4'
        };
        this.endpoint = S3_ENDPOINT;
        this.s3 = new aws_sdk_1.S3(s3Config);
        this.s3Stream = s3Stream(this.s3);
        this.log = (0, log_1.createLog)('files3');
        this.bucket = bucket;
        this.writeable = writeable;
    }
    /**
     * Create a SHA256 hash of any data provided.
     **/
    hash(data) {
        return (0, crypto_1.createHash)('sha256').update(data).digest('base64');
    }
    /**
     * Check if file exists
     **/
    async exists(path) {
        try {
            await (0, promises_1.access)(path);
            return true;
        }
        catch {
            return false;
        }
    }
    /**
     * Read file from disk as buffer and create hash of the data.
     **/
    async hashFile(filePath) {
        let data;
        try {
            data = await (0, promises_1.readFile)(filePath);
        }
        catch (err) {
            this.log.error(err);
        }
        return this.hash(data);
    }
    /**
     * create a file object on an S3 bucket and upload data.
     * Reads into memory
     **/
    async create(file, keyName) {
        if (!this.writeable)
            return false;
        const id = (0, uuid_1.v4)();
        const ext = mime.extension(file.mimetype);
        const key = typeof keyName !== 'undefined' ? keyName : `${id}.${ext}`;
        const webPath = (0, path_1.join)('/files/', this.bucket, key);
        const publicPath = (0, path_1.join)(`${this.bucket}.${this.endpoint}`, key);
        const record = {
            id,
            created: +new Date(),
            name: file.originalname,
            public: publicPath,
            path: webPath,
            path_hash: this.hash(webPath),
            hash: null,
            type: file.mimetype,
            size: null
        };
        const params = {
            Bucket: this.bucket,
            Key: key,
            Body: null
        };
        params.Body = file.buffer;
        record.hash = this.hash(file.buffer);
        record.size = file.buffer.byteLength;
        return new Promise((resolve, reject) => {
            return this.s3.putObject(params, (err, data) => {
                if (err) {
                    this.log.error('create', err);
                    return reject(err);
                }
                else {
                    this.log.info(`Saved file ${record.path}`);
                    return resolve(record);
                }
            });
        });
    }
    /*
     *  Create an s3 record using only a path reference of a local file
     *  Reads into memory
     */
    async createFromPath(filePath, keyName) {
        const filename = (0, path_1.basename)(filePath);
        const mimetype = mime.lookup(filePath);
        const key = typeof keyName !== 'undefined' ? keyName : null;
        let file;
        let buffer;
        try {
            buffer = await (0, promises_1.readFile)(filePath);
        }
        catch (err) {
            this.log.error('createFromPath', err);
        }
        file = {
            buffer,
            mimetype,
            originalname: filename
        };
        return this.create(file, key);
    }
    /**
     *  Create an s3 record from a stream
     *  Pass "createReadStream('')" object
     **/
    /*public async createStream (file : any) {
        if (!this.writeable) return false;
        const id : string = uuid();
        const ext : string | false = mime.extension(file.mimetype);
        const key : string = `${id}.${ext}`;
        const webPath : string = pathJoin('/files/', this.bucket, key);
        const publicPath : string = pathJoin(`${this.bucket}.${this.endpoint}`, key);
        const record : FileRecord = {
            id,
            created : +new Date(),
            name : file.originalname,
            public : publicPath,
            path : webPath,
            path_hash : this.hash(webPath),
            hash : null,
            type : file.mimetype,
            size : null
        };
        const params : S3Params = {
            Bucket: this.bucket,
            Key: key
        };
        const upload = this.s3Stream.upload(params);

        upload.maxPartSize(20971520); // 20 MB
        upload.concurrentParts(5);

        return new Promise((resolve : Function, reject : Function) => {
            upload.on('error', (err : Error) => {
                return reject(err);
            });
            upload.on('part', (details : any) => {
                this.log.info(`${details.ETag} - part: ${details.PartNumber} received: ${details.receivedSize} uploaded: ${details.uploadedSize}`)
            });
            upload.on('uploaded', (details : any) => {
                record.hash = details.ETag;
                record.size = details.uploadedSize;
                this.log.info(`Saved file ${record.path}`);
                return resolve(record);
            });
            this.log.info(`Streaming ${record.path} to S3`);
            stream.pipe(upload);
        });
    }*/
    /**
     * Create a stream . Bind to busboy.on('file', files3.createStream)
     * ex. (express POST route callback)
     * var busboy = new Busboy({ headers: req.headers });
     * busboy.on('file', files3.createStream)
     * req.pipe(busboy);
     **/
    /* public async createStreamExpress (fieldname : string, file : any, filename : string, encoding : any, mimetype : string) {
         if (!this.writeable) return false;
         const id : string = uuid();
         const ext : string | false = mime.extension(mimetype);
         const key : string = `${id}.${ext}`;
         const webPath : string = pathJoin('/files/', this.bucket, key);
         const publicPath : string = pathJoin(`${this.bucket}.${this.endpoint}`, key);
         const record : FileRecord = {
             id,
             created : +new Date(),
             name : filename,
             public : publicPath,
             path : webPath,
             path_hash : this.hash(webPath),
             hash : null,
             type : file.mimetype,
             size : null
         };
         const params : S3Params = {
             Bucket: this.bucket,
             Key: key
         };
         const upload = this.s3Stream.upload(params);
 
         upload.maxPartSize(20971520); // 20 MB
         upload.concurrentParts(5);
 
         return new Promise((resolve : Function, reject : Function) => {
             var s3 = new AWS.S3({
          params: {Bucket: 'sswa', Key: filename, Body: file},
          options: {partSize: 5 * 1024 * 1024, queueSize: 10}   // 5 MB
       });
       s3.upload().on('httpUploadProgress', function (evt) {
          this.log.info(evt);
       }).send(function (err, data) {
          s3UploadFinishTime = new Date();
          if(busboyFinishTime && s3UploadFinishTime) {
             res.json({
                uploadStartTime: uploadStartTime,
                busboyFinishTime: busboyFinishTime,
                s3UploadFinishTime: s3UploadFinishTime
             });
          }
          this.log.info(err, data);
       });
             file.on('data', ( data : any ) => {
                 upload.on('error', (err : Error) => {
                     return reject(err);
                 });
                 upload.on('part', (details : any) => {
                     this.log.info(`${details.ETag} - part: ${details.PartNumber} received: ${details.receivedSize} uploaded: ${details.uploadedSize}`)
                 });
                 upload.on('uploaded', (details : any) => {
                     record.hash = details.ETag;
                     record.size = details.uploadedSize;
                     this.log.info(`Saved file ${record.path}`)
                     return resolve(record)
                 });
                 data.pipe(upload);
             })
         });
     }*/
    /**
     * Create a stream from a path on the local device
     *
     * @param {string} filePath     Path to file
     * @param {string} keyName     (optional) Predefined key
     **/
    async createStreamFromPath(filePath, keyName) {
        if (!this.writeable)
            return false;
        const id = (0, uuid_1.v4)();
        const fileName = (0, path_1.basename)(filePath);
        const mimetype = mime.lookup(filePath);
        const ext = mime.extension(fileName);
        const key = typeof keyName !== 'undefined' ? keyName : `${id}.${ext}`;
        const webPath = (0, path_1.join)('/files/', this.bucket, key);
        const publicPath = (0, path_1.join)(`${this.bucket}.${this.endpoint}`, key);
        const record = {
            id,
            created: +new Date(),
            name: fileName,
            public: publicPath,
            path: webPath,
            path_hash: this.hash(webPath),
            hash: null,
            type: mimetype ? mimetype : null,
            size: null
        };
        const params = {
            Bucket: this.bucket,
            Key: key
        };
        const upload = this.s3Stream.upload(params);
        const stream = (0, fs_1.createReadStream)(filePath);
        upload.maxPartSize(20971520); // 20 MB
        upload.concurrentParts(5);
        return new Promise((resolve, reject) => {
            upload.on('error', (err) => {
                this.log.error('createStreamFromPath', err);
                return reject(err);
            });
            upload.on('part', (details) => {
                this.log.info(`${details.ETag} - part: ${details.PartNumber} received: ${details.receivedSize} uploaded: ${details.uploadedSize}`);
            });
            upload.on('uploaded', (details) => {
                record.hash = details.ETag;
                record.size = details.uploadedSize;
                this.log.info(`Saved file ${record.path}`);
                return resolve(record);
            });
            this.log.info(`Streaming ${record.path} to S3`);
            stream.pipe(upload);
        });
    }
    /**
     * Read a file from S3 using a key
     *
     * @param {string} key File key
     *
     * @returns {string} File data
     **/
    async read(key) {
        const params = {
            Bucket: this.bucket,
            Key: key
        };
        return new Promise((resolve, reject) => {
            return this.s3.getObject(params, (err, data) => {
                if (err) {
                    this.log.error('read', err);
                    return reject(err);
                }
                return resolve(data.Body); //buffer
            });
        });
    }
    /*
        const xhr = new XMLHttpRequest();
        xhr.open('PUT', signedUrl);
        xhr.setRequestHeader('Content-Type', file.type);
        xhr.setRequestHeader('x-amz-acl', 'public-read');
        xhr.send(file);
    */
    /**
     * Get a signed put key for writing
     *
     * @param {string} key          Key that file will be located at
     * @param {string} fileType     Mimetype of file
     *
     * @returns {string} Url of signed key
     **/
    async signedPutKey(key, fileType) {
        const s3Params = {
            Bucket: this.bucket,
            Key: key,
            ContentType: fileType,
            Expires: new Date((new Date()).getTime() + 24 * 60 * 60 * 1000) //1 day
            //ACL: 'public-read',
        };
        return new Promise((resolve, reject) => {
            return this.s3.getSignedUrl('putObject', s3Params, (err, url) => {
                if (err) {
                    this.log.error('signedPutKey', err);
                    return reject(err);
                }
                return resolve(url);
            });
        });
    }
    /**
     * Get a signed read key for writing
     *
     * @param {string} key          Key that file will be located at
     *
     * @returns {string} Url of signed key
     **/
    async signedGetKey(key) {
        const s3Params = {
            Bucket: this.bucket,
            Key: key,
            //Expires: new Date((new Date()).getTime() + 24 * 60 * 60 * 1000) //1 day
            //ACL: 'public-read',
            //Expires: 60 ?
        };
        return new Promise((resolve, reject) => {
            return this.s3.getSignedUrl('getObject', s3Params, (err, url) => {
                if (err) {
                    this.log.error('signedGetKey', err);
                    return reject(err);
                }
                return resolve(url);
            });
        });
    }
    /*
    readStream (to express or server)
    s3.getObject(params)
    .on('httpHeaders', function (statusCode, headers) {
        res.set('Content-Length', headers['content-length']);
        res.set('Content-Type', headers['content-type']);
        this.response.httpResponse.createUnbufferedStream()
            .pipe(res);
    })
    .send();
--------
    var fileStream = fs.createWriteStream('/path/to/file.jpg');
    var s3Stream = s3.getObject({Bucket: 'myBucket', Key: 'myImageFile.jpg'}).createReadStream();

    // Listen for errors returned by the service
    s3Stream.on('error', function(err) {
        // NoSuchKey: The specified key does not exist
        this.log.error(err);
    });

    s3Stream.pipe(fileStream).on('error', function(err) {
        // capture any errors that occur when writing data to the file
        this.log.error('File Stream:', err);
    }).on('close', function() {
        this.log.info('Done.');
    });
    */
    /**
     * Delete an object at a specific key
     *
     * @param {string} key Key for object
     *
     * @returns {boolean} True if successful
     **/
    async delete(key) {
        if (!this.writeable)
            return false;
        const params = {
            Bucket: this.bucket,
            Key: key
        };
        return new Promise((resolve, reject) => {
            return this.s3.deleteObject(params, (err, data) => {
                if (err) {
                    this.log.error('delete', err);
                    return reject(err);
                }
                return resolve(true); //buffer
            });
        });
    }
    /**
     * Lists all objects with a specific prefix
     **/
    async list(prefix) {
        const params = {
            Bucket: this.bucket,
            Prefix: prefix
        };
        return new Promise((resolve, reject) => {
            return this.s3.listObjectsV2(params, (err, data) => {
                if (err) {
                    this.log.error('list', err);
                    return reject(err);
                }
                return resolve(data);
            });
        });
    }
    async update() {
        if (!this.writeable)
            return false;
    }
}
exports.Files3 = Files3;
module.exports = { Files3 };
//# sourceMappingURL=index.js.map