Add support for S3 backed uploads

2020-11-15 10:55:36 -06:00 · 2020-11-15 10:55:36 -06:00 · f7e4c535f8
commit f7e4c535f8
parent c569840618
4 changed files with 149 additions and 15 deletions
--- a/config/upload.config.js
+++ b/config/upload.config.js
@ -8,7 +8,7 @@ const upload_config = {
    /*
     * The name of the upload backend to use by default.
     */
-    default_store: 'flitter',
+    default_store: env('UPLOAD_DEFAULT_STORE', 'flitter'),
    enabled: true,

    /*
@ -30,6 +30,30 @@ const upload_config = {
            // path of the application.
            destination: './uploads',
        },
+
+        /*
+         * Example of an Amazon AWS S3 (or compatible) uploader.
+         */
+        s3: {
+            enabled: true,
+
+            // Backed by an S3 bucket.
+            type: 'AWSS3Store',
+
+            // AWS SDK credentials and configuration
+            aws: {
+                region: env('AWS_REGION', 'us-east-1'),
+                key_id: env('AWS_ACCESS_KEY_ID'),
+                key_secret: env('AWS_SECRET_ACCESS_KEY'),
+                endpoint: env('AWS_S3_ENDPOINT'),
+            },
+
+            // The bucket to which the files should be uploaded
+            bucket: env('AWS_UPLOAD_BUCKET', 'flitter.localhost'),
+
+            // The prefix to be used for file uploads
+            prefix: env('AWS_UPLOAD_PREFIX', ''),
+        },
    },
 }

--- a/example.env
+++ b/example.env
@ -26,4 +26,12 @@ AUTH_OAUTH2_CLIENT_SECRET=
 #starship coreid login config
 AUTH_COREID_ENABLE=true
 AUTH_COREID_CLIENT_ID=
-AUTH_COREID_CLIENT_SECRET=
+AUTH_COREID_CLIENT_SECRET=
+
+#MinIO/S3 upload config
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_S3_ENDPOINT="http://nfs.platform.local:9000"
+AWS_UPLOAD_BUCKET="noded.localhost"
+AWS_UPLOAD_PREFIX="uploaded_files"
+UPLOAD_DEFAULT_STORE=s3
--- a/package.json
+++ b/package.json
@ -16,6 +16,7 @@
  "author": "Garrett Mills <garrett@glmdev.tech> (https://garrettmills.dev/)",
  "license": "MIT",
  "dependencies": {
+    "aws-sdk": "^2.792.0",
    "cors": "^2.8.5",
    "flitter-auth": "^0.19.1",
    "flitter-cli": "^0.16.0",
@ -23,13 +24,14 @@
    "flitter-flap": "^0.5.2",
    "flitter-forms": "^0.8.1",
    "flitter-orm": "^0.4.0",
-    "flitter-upload": "^0.8.1",
+    "flitter-upload": "^0.9.2",
    "jsonwebtoken": "^8.5.1",
-    "libflitter": "^0.57.1",
+    "libflitter": "^0.58.1",
    "markdown": "^0.5.0",
    "ncp": "^2.0.0",
    "rimraf": "^3.0.2",
    "tar": "^6.0.5",
+    "tempfile": "^3.0.0",
    "tmp": "^0.2.1"
  }
 }
--- a/yarn.lock
+++ b/yarn.lock
@ -486,6 +486,21 @@ ast-types@0.9.6:
  resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.9.6.tgz#102c9e9e9005d3e7e3829bf0c4fa24ee862ee9b9"
  integrity sha1-ECyenpAF0+fjgpvwxPok7oYu6bk=

+aws-sdk@^2.792.0:
+  version "2.792.0"
+  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.792.0.tgz#d124a6074244a4675e0416887734e8f6934bdd30"
+  integrity sha512-h7oSlrCDtZkW5qNw/idKmMjjNJaaPlXFY+NbqtaTjejpCyVuIonUmFvm8GW16V58Avj/hujJfhpX9q0BMCg+VQ==
+  dependencies:
+    buffer "4.9.2"
+    events "1.1.1"
+    ieee754 "1.1.13"
+    jmespath "0.15.0"
+    querystring "0.2.0"
+    sax "1.2.1"
+    url "0.10.3"
+    uuid "3.3.2"
+    xml2js "0.4.19"
+
 axios@^0.19.0:
  version "0.19.0"
  resolved "https://registry.yarnpkg.com/axios/-/axios-0.19.0.tgz#8e09bff3d9122e133f7b8101c8fbdd00ed3d2ab8"
@ -665,6 +680,11 @@ balanced-match@^1.0.0:
  resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
  integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c=

+base64-js@^1.0.2:
+  version "1.5.1"
+  resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
+  integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
+
 basic-auth@~0.0.1:
  version "0.0.1"
  resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-0.0.1.tgz#31ddb65843f6c35c6fea7beb46a987cb8ce18924"
@ -757,6 +777,15 @@ buffer-equal-constant-time@1.0.1:
  resolved "https://registry.yarnpkg.com/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz#f8e71132f7ffe6e01a5c9697a4c6f3e48d5cc819"
  integrity sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=

+buffer@4.9.2:
+  version "4.9.2"
+  resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8"
+  integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==
+  dependencies:
+    base64-js "^1.0.2"
+    ieee754 "^1.1.4"
+    isarray "^1.0.0"
+
 bunyan@1.3.3:
  version "1.3.3"
  resolved "https://registry.yarnpkg.com/bunyan/-/bunyan-1.3.3.tgz#bf4e301c1f0bf888ec678829531f7b5d212e9e81"
@ -1442,6 +1471,11 @@ etag@~1.8.1:
  resolved "https://registry.yarnpkg.com/etag/-/etag-1.8.1.tgz#41ae2eeb65efa62268aebfea83ac7d79299b0887"
  integrity sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=

+events@1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
+  integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=
+
 express-busboy@^7.0.1:
  version "7.0.1"
  resolved "https://registry.yarnpkg.com/express-busboy/-/express-busboy-7.0.1.tgz#44c750b2475f5ae3ceb6b490097da87e540e463a"
@ -1671,14 +1705,16 @@ flitter-orm@^0.4.0:
    sinon "^9.0.0"
    uuid "^3.4.0"

-flitter-upload@^0.8.1:
-  version "0.8.1"
-  resolved "https://registry.yarnpkg.com/flitter-upload/-/flitter-upload-0.8.1.tgz#c69c2b16f8f5227532fe28373de4c1f8094fcdf6"
-  integrity sha512-ASLIAibIriSOes3zODcD40+Ly5mGATvAVf41bOZDTRjnTT6ZG8PTbV3EaU+WerPtIO0cj8ccPiusR8X/nPkKeA==
+flitter-upload@^0.9.2:
+  version "0.9.2"
+  resolved "https://registry.yarnpkg.com/flitter-upload/-/flitter-upload-0.9.2.tgz#1cb7ab9ad5f70be83b1a97d67822480462a0bc3f"
+  integrity sha512-MDQozy/WQKNAG6Zp02mOo8thKdImN4o6mRpoH+4U6wDYnIGbnbDFBiHHEHT977bHhDjjswr5t3zo6tvwwzMSrA==
  dependencies:
+    aws-sdk "^2.792.0"
    es6-promisify "^6.0.1"
    mkdirp "^1.0.3"
    ncp "^2.0.0"
+    tempfile "^3.0.0"
    uuid "^3.3.2"

 follow-redirects@1.5.10:
@ -1991,6 +2027,16 @@ iconv-lite@0.4.24, iconv-lite@^0.4.4, iconv-lite@^0.4.5:
  dependencies:
    safer-buffer ">= 2.1.2 < 3"

+ieee754@1.1.13:
+  version "1.1.13"
+  resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84"
+  integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==
+
+ieee754@^1.1.4:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
+  integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==
+
 ignore-walk@^3.0.1:
  version "3.0.1"
  resolved "https://registry.yarnpkg.com/ignore-walk/-/ignore-walk-3.0.1.tgz#a83e62e7d272ac0e3b551aaa82831a19b69f82f8"
@ -2202,7 +2248,7 @@ isarray@0.0.1:
  resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf"
  integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=

-isarray@~1.0.0:
+isarray@^1.0.0, isarray@~1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
  integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=
@ -2278,6 +2324,11 @@ iterall@^1.2.2:
  resolved "https://registry.yarnpkg.com/iterall/-/iterall-1.2.2.tgz#92d70deb8028e0c39ff3164fdbf4d8b088130cd7"
  integrity sha512-yynBb1g+RFUPY64fTrFv7nsjRrENBQJaX2UL+2Szc9REFrSNm1rpSXHGzhmAy7a9uv3vlvgBlXnf9RqmPH1/DA==

+jmespath@0.15.0:
+  version "0.15.0"
+  resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217"
+  integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=
+
 js-beautify@^1.10.2:
  version "1.10.2"
  resolved "https://registry.yarnpkg.com/js-beautify/-/js-beautify-1.10.2.tgz#88c9099cd6559402b124cfab18754936f8a7b178"
@ -2455,10 +2506,10 @@ leven@^1.0.2:
  resolved "https://registry.yarnpkg.com/leven/-/leven-1.0.2.tgz#9144b6eebca5f1d0680169f1a6770dcea60b75c3"
  integrity sha1-kUS27ryl8dBoAWnxpncNzqYLdcM=

-libflitter@^0.57.1:
-  version "0.57.1"
-  resolved "https://registry.yarnpkg.com/libflitter/-/libflitter-0.57.1.tgz#e605333a5e38cadac1203e4be8ea685c5520472b"
-  integrity sha512-bEu02HZjcAp53A2xnE0hz/LVhkErvnqaKmlNOqvJJaGMjKClfVuqJlr9bFfWU84qXL9SaHWd06P3nxIVKTvFIw==
+libflitter@^0.58.1:
+  version "0.58.1"
+  resolved "https://registry.yarnpkg.com/libflitter/-/libflitter-0.58.1.tgz#bee2e5e65a325046632c799f76c455cb5a2f203e"
+  integrity sha512-23k/G6HfMEHvGxjIupVOjpmLkcLYYDRvXSYTLkDMeAyGfzXNTSVVS2a+TFoR72Nuj3dMrDxIqvHp1iNGn4y54Q==
  dependencies:
    colors "^1.3.3"
    connect-mongodb-session "^2.2.0"
@ -3459,6 +3510,11 @@ pug@^2.0.3:
    pug-runtime "^2.0.4"
    pug-strip-comments "^1.0.3"

+punycode@1.3.2:
+  version "1.3.2"
+  resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d"
+  integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=
+
 q@^1.1.2:
  version "1.5.1"
  resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7"
@ -3479,6 +3535,11 @@ qs@^6.9.0:
  resolved "https://registry.yarnpkg.com/qs/-/qs-6.9.1.tgz#20082c65cb78223635ab1a9eaca8875a29bf8ec9"
  integrity sha512-Cxm7/SS/y/Z3MHWSxXb8lIFqgqBowP5JMlTUFyJN88y0SGQhVmZnqFK/PeuMX9LzUyWsqqhNxIyg0jlzq946yA==

+querystring@0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
+  integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=
+
 random-bytes@~1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/random-bytes/-/random-bytes-1.0.0.tgz#4f68a1dc0ae58bd3fb95848c30324db75d64360b"
@ -3768,7 +3829,12 @@ saslprep@^1.0.0:
  dependencies:
    sparse-bitfield "^3.0.3"

-sax@^1.2.4:
+sax@1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a"
+  integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o=
+
+sax@>=0.6.0, sax@^1.2.4:
  version "1.2.4"
  resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
  integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
@ -4160,6 +4226,19 @@ tar@^6.0.5:
    mkdirp "^1.0.3"
    yallist "^4.0.0"

+temp-dir@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/temp-dir/-/temp-dir-2.0.0.tgz#bde92b05bdfeb1516e804c9c00ad45177f31321e"
+  integrity sha512-aoBAniQmmwtcKp/7BzsH8Cxzv8OL736p7v1ihGb5e9DJ9kTwGWHrQrVB5+lfVDzfGrdRzXch+ig7LHaY1JTOrg==
+
+tempfile@^3.0.0:
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/tempfile/-/tempfile-3.0.0.tgz#5376a3492de7c54150d0cc0612c3f00e2cdaf76c"
+  integrity sha512-uNFCg478XovRi85iD42egu+eSFUmmka750Jy7L5tfHI5hQKKtbPnxaSaXAbBqCDYrw3wx4tXjKwci4/QmsZJxw==
+  dependencies:
+    temp-dir "^2.0.0"
+    uuid "^3.3.2"
+
 test-exclude@^6.0.0:
  version "6.0.0"
  resolved "https://registry.yarnpkg.com/test-exclude/-/test-exclude-6.0.0.tgz#04a8698661d805ea6fa293b6cb9e63ac044ef15e"
@ -4292,6 +4371,14 @@ unpipe@1.0.0, unpipe@~1.0.0:
  resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
  integrity sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=

+url@0.10.3:
+  version "0.10.3"
+  resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64"
+  integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=
+  dependencies:
+    punycode "1.3.2"
+    querystring "0.2.0"
+
 user-home@^1.1.1:
  version "1.1.1"
  resolved "https://registry.yarnpkg.com/user-home/-/user-home-1.1.1.tgz#2b5be23a32b63a7c9deb8d0f28d485724a3df190"
@ -4307,7 +4394,7 @@ utils-merge@1.0.1:
  resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
  integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=

-uuid@^3.3.2:
+uuid@3.3.2, uuid@^3.3.2:
  version "3.3.2"
  resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131"
  integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==
@ -4445,6 +4532,19 @@ write-file-atomic@^3.0.0:
    signal-exit "^3.0.2"
    typedarray-to-buffer "^3.1.5"

+xml2js@0.4.19:
+  version "0.4.19"
+  resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7"
+  integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==
+  dependencies:
+    sax ">=0.6.0"
+    xmlbuilder "~9.0.1"
+
+xmlbuilder@~9.0.1:
+  version "9.0.7"
+  resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d"
+  integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=
+
 y18n@^3.2.0:
  version "3.2.1"
  resolved "https://registry.yarnpkg.com/y18n/-/y18n-3.2.1.tgz#6d15fba884c08679c0d77e88e7759e811e07fa41"