Delete all items in Dynamodb using Lambda?

Question

Using Lambda (node.js) - how to delete all the items in the Dynamodb table?

There are 500K rows in the table

I have tried using scan method and then loop through each item and then using delete method. It only allow up to 3000 rows only.

Code

exports.handler = function(context, callback) {
  getRecords().then((data) => {
    data.Items.forEach(function(item) {
      deleteItem(item.Id).then((data1) => {

      });
    });
  });
};

var deleteItem = function(id) {
  var params = {
    TableName: "TableName",
    Key: {
      "Id": id
    },
  };

  return new Promise(function(resolve, reject) {
    client.delete(params, function(err, data) {
      if (err) {
        reject(err);
      } else {
        resolve();
      }
    });
  });
}


function getRecords() {
  var params = {
    TableName: 'TableName',
    IndexName: 'Type-index',
    KeyConditionExpression: 'Type = :ty',
    ExpressionAttributeValues: {
      ':ty': "1"
    },
    ProjectionExpression: "Id",
  };

  return new Promise(function(resolve, reject) {
    client.query(params, function(err, data) {
      if (err) {
        reject(err);
      } else {
        resolve(data);
      }
    });
  });
}

You want to empty the table? Lambda is not the best choice for this. Rather than scan/delete every item, a better option is often to simply delete and then re-create the table. — jarmod
– jarmod, Commented Jun 29, 2018 at 23:33

Krishna Pankhania · Accepted Answer · 2021-06-11 09:12:11Z

7

There is already one right answer, but here is another code snippet to delete all records from Dynamo DB.

const AWS = require("aws-sdk");

AWS.config.update({
  region: "us-east-1",
});

const docClient = new AWS.DynamoDB.DocumentClient();
const getAllRecords = async (table) => {
  let params = {
    TableName: table,
  };
  let items = [];
  let data = await docClient.scan(params).promise();
  items = [...items, ...data.Items];
  while (typeof data.LastEvaluatedKey != "undefined") {
    params.ExclusiveStartKey = data.LastEvaluatedKey;
    data = await docClient.scan(params).promise();
    items = [...items, ...data.Items];
  }
  return items;
};
const deleteItem = (table, id) => {
  var params = {
    TableName: table,
    Key: {
      id: id,
    },
  };

  return new Promise(function (resolve, reject) {
    docClient.delete(params, function (err, data) {
      if (err) {
        console.log("Error Deleting ", id,err);
        reject(err);
      } else {
        console.log("Success Deleting ", id,err);
        resolve();
      }
    });
  });
};
exports.handler = async function (event, context, callback) {
  try {
    const tableName = "<table>";
    // scan and get all items
    const allRecords = await getAllRecords(tableName);
    // delete one by one 
    for (const item of allRecords) {
      await deleteItem(tableName, item.id);
    }
    callback(null, {
      msg: "All records are deleted.",
    });
  } catch (e) {
    callback(null, JSON.stringify(e, null, 2));
  }
};

answered Jun 11, 2021 at 9:12

Krishna Pankhania

1011 silver badge3 bronze badges

Sign up to request clarification or add additional context in comments.

1 Comment

Peter Suwara Over a year ago

Great answer, with more modern JS than the previous answer. Should be the best answer at this point.

Renato Byrro · Accepted Answer · 2022-11-09 20:27:53Z

3

A Scan operation consumes Read capacity. Each Read returns up to 4 kb of data. When this limit is reached, the Scan returns only what it has found until there. If you need more, you need to issue another Scan request.

This, you'll need two loops: 1) loop to delete all records returned at each Scan; 2) loop to keep scanning multiple times, until you reach the end of the table

Make sure you use consistent Reads or wait 1 or 2 second(s) before issuing another Scan, otherwise you may get repeated items in different Scans.

exports.handler = function(context, callback) {
  clearRecords();
};

clearRecords = function() {
  getRecords().then((data) => {
    data.Items.forEach(function(item) {
      deleteItem(item.Id).then((data1) => {});
    });
    clearRecords(); // Will call the same function over and over
  });
}

Observe that Lambda has a timeout limit of 15 minutes. Since you have 500K items in your table, it's likely that your Lambda will timeout and you'll need to trigger it more than once. You could also make your Lambda call itself after 14:50, for example, just take a look at the AWS SDK documentation for triggering Lambda functions. For this matter, you might also want to check the getRemainingTimeInMillis() method from the context object.

edited Nov 9, 2022 at 20:27

answered Jun 29, 2018 at 23:26

Renato Byrro

3,87422 silver badges37 bronze badges

5 Comments

I'll-Be-Back Over a year ago

Would Query (selecting specific field with index) be cheaper than using Scan? If you don't mind posting code, it would be helpful for me and other

Renato Byrro Over a year ago

Dynamo doesn't charge differently by type of Read, only by how many Reads you consume. Currently, 4kb of data consumes one Read (or half Read, if not consistent). Query and Scan use Read capacity in the same way. Queries are cheaper only when you need a subset of the table, it's a way to get directly to the items you're interested in without the need to Read the rest of the Table. Since you need to get all items anyway, using a Query or Scan will consume the same amount of Read capacity, this same amount of money. As for code, please post yours and we can start from there?

Renato Byrro Over a year ago

Added a suggestion to my answer. You could remove the logic to outside the handler and make it call itself after each execution. The clearRecords function could also be called multiple times in parallel, but it would increase the reserved throughput needed in your DynamoDB table.

Renato Byrro Over a year ago

Modified to move the clearRecords call outside the deleteItem loop, so that it will be called only once at the end of each getRecords call.

Stephen Over a year ago

Since this article was written, lambda can now have upto a 15minute timeout....

Collectives™ on Stack Overflow

Delete all items in Dynamodb using Lambda?

2 Answers 2

1 Comment

5 Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

1 Comment

5 Comments

Your Answer

Sign up or log in

Post as a guest

Related