diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | README.md | 99 | ||||
| -rw-r--r-- | handler.js | 26 | ||||
| -rw-r--r-- | package.json | 11 | ||||
| -rw-r--r-- | serverless.yml | 23 |
5 files changed, 161 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2c44480 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +node_modules +.serverless diff --git a/README.md b/README.md new file mode 100644 index 0000000..f4b792c --- /dev/null +++ b/README.md @@ -0,0 +1,99 @@ +<!-- +title: AWS Fetch image from URL and upload to S3 example in NodeJS +description: This example display how to fetch an image from remote source (URL) and then upload this image to a S3 bucket. +layout: Doc +--> +# Fetch image from URL then upload to s3 Example + +This example display how to fetch an image from remote source (URL) and then upload this image to a S3 bucket. + +## Use-cases + +- Store a user's profile picture from another service. + +## How it works + +We first fetch the data from given url and then call the S3 API `putObject` to upload it to the bucket. + +```js +fetch('image URL') + .then(res => { + return s3.putObject({Bucket, Key, Body: res.body}).promise(); + }).then(res => { + callback(null, res); + }).catch(err => { + callback(err, null); + }); +``` + +## Setup + +Since this plugin uses the Serverless plugin `serverless-secrets-plugin` you need to setup the `node_modules` by running: + +```bash +npm install +``` + +In addition you need to create an S3 bucket you want to store the files in. After you created the bucket add change the bucket name in `serverless.yml` custom settings to your buckets. + +```yml +custom: + bucket: <your-bucket-name> +``` + +## Deploy + +In order to deploy the you endpoint simply run + +```bash +serverless deploy +``` + +The expected result should be similar to: + +```bash +Serverless: Creating Stack... +Serverless: Checking Stack create progress... +..... +Serverless: Stack create finished... +Serverless: Packaging service... +Serverless: Uploading CloudFormation file to S3... +Serverless: Uploading service .zip file to S3 (1.8 KB)... +Serverless: Updating Stack... +Serverless: Checking Stack update progress... +................ +Serverless: Stack update finished... + +Service Information +service: aws-node-fetch-file-and-store-in-s3 +stage: dev +region: us-west-1 +api keys: + None +endpoints: + None +functions: + aws-node-fetch-file-and-store-in-s3-dev-save: arn:aws:lambda:us-west-1:377024778620:function:aws-node-fetch-file-and-store-in-s3-dev-save +``` + +## Usage + +You can now send an HTTP request directly to the endpoint using a tool like curl + +```bash +serverless invoke --function save --log --data='{ "image_url": "https://assets-cdn.github.com/images/modules/open_graph/github-mark.png", "key": "github.png"}' +``` + +The expected result should be similar to: + +```bash +"Saved" +-------------------------------------------------------------------- +START RequestId: c658859d-bd42-11e6-ac1f-c7a7ee5bd7f3 Version: $LATEST +END RequestId: c658859d-bd42-11e6-ac1f-c7a7ee5bd7f3 +REPORT RequestId: c658859d-bd42-11e6-ac1f-c7a7ee5bd7f3 Duration: 436.94 ms Billed Duration: 500 ms Memory Size: 1024 MB Max Memory Used: 29 MB +``` + +## Scaling + +By default, AWS Lambda limits the total concurrent executions across all functions within a given region to 100. The default limit is a safety limit that protects you from costs due to potential runaway or recursive functions during initial development and testing. To increase this limit above the default, follow the steps in [To request a limit increase for concurrent executions](http://docs.aws.amazon.com/lambda/latest/dg/concurrent-executions.html#increase-concurrent-executions-limit). diff --git a/handler.js b/handler.js new file mode 100644 index 0000000..4f5c9cb --- /dev/null +++ b/handler.js @@ -0,0 +1,26 @@ +'use strict'; + +const fetch = require('node-fetch'); +const AWS = require('aws-sdk'); // eslint-disable-line import/no-extraneous-dependencies + +const s3 = new AWS.S3(); + +module.exports.save = (event, context, callback) => { + fetch(event.image_url) + .then((response) => { + if (response.ok) { + return response; + } + return Promise.reject(new Error( + `Failed to fetch ${response.url}: ${response.status} ${response.statusText}`)); + }) + .then(response => response.buffer()) + .then(buffer => ( + s3.putObject({ + Bucket: process.env.BUCKET, + Key: event.key, + Body: buffer, + }).promise() + )) + .then(v => callback(null, v), callback); +}; diff --git a/package.json b/package.json new file mode 100644 index 0000000..8163755 --- /dev/null +++ b/package.json @@ -0,0 +1,11 @@ +{ + "name": "aws-fetch-file-and-store-in-s3", + "description": "Fetch an image from remote source (URL) and then upload the image to a S3 bucket.", + "version": "1.0.0", + "author": "Bozhao Yu", + "license": "MIT", + "dependencies": { + "aws-sdk": "^2.7.9", + "node-fetch": "^1.6.3" + } +} diff --git a/serverless.yml b/serverless.yml new file mode 100644 index 0000000..44fa378 --- /dev/null +++ b/serverless.yml @@ -0,0 +1,23 @@ +service: fetch-file-and-store-in-s3 + +frameworkVersion: ">=1.1.0" + +custom: + bucket: <your-bucket-name> + +provider: + name: aws + runtime: nodejs4.3 + stage: dev + region: us-west-1 + iamRoleStatements: + - Effect: Allow + Action: + - s3:PutObject + Resource: "arn:aws:s3:::${self:custom.bucket}/*" + +functions: + save: + handler: handler.save + environment: + BUCKET: ${self:custom.bucket} |