AWS DynamoDB Cross Region Replication using Lambda (Node.js)

Step:1 Create simple table between two different region:

@us-east-1

Table Name:  master

Primary Key – ID : “String”

Sort Key – Name: “String”

@us-west-2

Table Name:  west

Primary Key – ID : “String”

Sort Key – Name: “String”

 

Step:2 Create IAM assume role  for LAMBDA to access DynamoDB Table:

Create new IAM Role “lambda-dynamodb-execution-role” and attach the default policy “AmazonDynamoDBFullAccess

 

Step:3 Create lambda function “master-2-dr-replicaiton” using below code:

Code: (node.js 4.3)

var startTime = Date.now();
console.log(“Start Time: ” + startTime);
var http = require(‘http’);
var https = require(‘https’);
http.globalAgent.maxSockets = 500;
https.globalAgent.maxSockets = 500;

var destinationRegion = ‘us-west-2′;
var destinationTable = ‘west';

//Set up AWS client
var AWS = require(‘aws-sdk’);

//Update AWS configuration to set region
AWS.config.update({region : destinationRegion});

// Set up DynamoDB client
var dynamodb = new AWS.DynamoDB();

exports.handler = function(event, context) {
    var handlerStart = Date.now();
    console.log(“Handler Start Time: ” + handlerStart);
    //Keep track of how may requests are in flight
    var inflightRequests = 0;

    //Deduplicate updates to the same key
    var buffer = {};
    event.Records.forEach(function(record){
        buffer[JSON.stringify(record.dynamodb.Keys)] = record.dynamodb;
    });

    // Make a callback function to execute once the request completes
    var handleResponse = function(err, data) {
        if (err) {
            //log errors
            console.error(err, err.stack);
        } else {
            //check if all requests are finished, if so, end the function
            inflightRequests–;
            if (inflightRequests === 0) {
                context.succeed(“Successfully processed” + event.Records.length + ” records. “);
                console.log(“Total Time: “, Date.now()-handlerStart, “ms”);
            }
        }
    }

    for (var key in buffer) {
        if (!buffer.hasOwnProperty(key)) continue;

        // Get the new image of the DynamoDB stream record
        var oldItemImage = buffer[key].OldImage;
        var newItemImage = buffer[key].NewImage;

        // Figure out the what type of request to send
        if (validate(oldItemImage) && !validate(newItemImage)) {
            dynamodb.deleteItem({Key : buffer[key].Keys, TableName : destinationTable}, handleResponse);
        } else if (validate(newItemImage)) {
            dynamodb.putItem({Item : newItemImage, TableName : destinationTable}, handleResponse);
        } else {
            console.error(“Both old and new images are not valid.”);
        }

        // Increase count for number of requests in flight
        inflightRequests++;
    }

    console.log(“Sent all request took: “, Date.now()-handlerStart, “ms”);

    // if there are no more requests pending, end the function
    if (inflightRequests === 0) {
        context.succeed(“Successfully processed ” + event.Records.length + ” records.”);
    }
};

// validate the given image of record, return true only if the image is valid
var validate = function(image) {
  if (typeof image !== ‘undefined’ && image) {
      return true;
  }
  return false;
};

 

Configuration:

Runtime = Node.js 4.3

Handler index.handler

Role –> ” lambda-dynamodb-execution-role”

 

Step:4 Add a “Trigger” in DynamoDB table “master” to map the above created lambda function

Step:5 Enable Stream on DynamoDB table “master”

Step:6 Insert some data on the “master” table and check the “dr” table.

Step:7 Also verify the cloud-watch logs / Lambda monitoring for any errors.

 

Leave a Comment

Your email address will not be published. Required fields are marked *

You may use these HTML tags and attributes: <a href="" title=""> <abbr title=""> <acronym title=""> <b> <blockquote cite=""> <cite> <code> <del datetime=""> <em> <i> <q cite=""> <s> <strike> <strong>