Building a RESTful API from Scratch with Node.js (Part 7)
APIs tutorialcreate an APIAPI development guideTutorial

6 min read

Adding Pagination to Your Node.js API

This tutorial builds on our tested API by implementing robust pagination for GET requests. We'll cover different pagination strategies, cursor-based pagination, and how to handle large datasets efficiently.

Prerequisites

  • Completed Parts 1-6 of the tutorial
  • Basic understanding of REST APIs
  • MongoDB and Node.js installed

Project Setup

Install the required dependencies:

npm install mongoose-cursor-pagination query-string

Updated Project Structure

Add these new files to your project:

books-api/
├── src/
│   ├── config/
│   │   └── pagination.js
│   ├── middleware/
│   │   └── pagination.js
│   ├── utils/
│   │   └── paginationHelpers.js
│   └── types/
│       └── pagination.js

Pagination Configuration

Create src/config/pagination.js:

module.exports = {
    // Default pagination settings
    DEFAULT_PAGE_SIZE: 10,
    MAX_PAGE_SIZE: 100,
    
    // Sorting defaults
    DEFAULT_SORT_FIELD: 'createdAt',
    DEFAULT_SORT_ORDER: 'desc',
    
    // Cursor settings
    CURSOR_ENCODING: 'base64',
    
    // Response field names
    META_FIELDS: {
        totalCount: 'totalCount',
        pageSize: 'pageSize',
        currentPage: 'currentPage',
        totalPages: 'totalPages',
        hasNextPage: 'hasNextPage',
        hasPrevPage: 'hasPrevPage',
        nextCursor: 'nextCursor',
        prevCursor: 'prevCursor'
    }
};

Pagination Middleware

Create src/middleware/pagination.js:

const { 
    DEFAULT_PAGE_SIZE,
    MAX_PAGE_SIZE,
    DEFAULT_SORT_FIELD,
    DEFAULT_SORT_ORDER
} = require('../config/pagination');

// Offset-based pagination middleware
const offsetPagination = (req, res, next) => {
    const page = parseInt(req.query.page) || 1;
    const limit = Math.min(
        parseInt(req.query.limit) || DEFAULT_PAGE_SIZE,
        MAX_PAGE_SIZE
    );
    const skip = (page - 1) * limit;
    const sortField = req.query.sortBy || DEFAULT_SORT_FIELD;
    const sortOrder = req.query.order || DEFAULT_SORT_ORDER;
    
    req.pagination = {
        page,
        limit,
        skip,
        sort: { [sortField]: sortOrder === 'desc' ? -1 : 1 }
    };
    
    next();
};

// Cursor-based pagination middleware
const cursorPagination = (req, res, next) => {
    const limit = Math.min(
        parseInt(req.query.limit) || DEFAULT_PAGE_SIZE,
        MAX_PAGE_SIZE
    );
    const cursor = req.query.cursor;
    const sortField = req.query.sortBy || DEFAULT_SORT_FIELD;
    const sortOrder = req.query.order || DEFAULT_SORT_ORDER;
    
    req.pagination = {
        limit,
        cursor,
        sort: { [sortField]: sortOrder === 'desc' ? -1 : 1 }
    };
    
    next();
};

module.exports = {
    offsetPagination,
    cursorPagination
};

Pagination Helpers

Create src/utils/paginationHelpers.js:

const { META_FIELDS, CURSOR_ENCODING } = require('../config/pagination');

// Helper for offset-based pagination
const createOffsetPaginationResponse = async (
    model,
    query,
    pagination,
    projection = {},
    populate = []
) => {
    const { page, limit, skip, sort } = pagination;
    
    // Execute queries in parallel
    const [data, totalCount] = await Promise.all([
        model
            .find(query, projection)
            .sort(sort)
            .skip(skip)
            .limit(limit)
            .populate(populate),
        model.countDocuments(query)
    ]);
    
    const totalPages = Math.ceil(totalCount / limit);
    
    return {
        data,
        meta: {
            [META_FIELDS.totalCount]: totalCount,
            [META_FIELDS.pageSize]: limit,
            [META_FIELDS.currentPage]: page,
            [META_FIELDS.totalPages]: totalPages,
            [META_FIELDS.hasNextPage]: page < totalPages,
            [META_FIELDS.hasPrevPage]: page > 1
        }
    };
};

// Helper for cursor-based pagination
const createCursorPaginationResponse = async (
    model,
    query,
    pagination,
    projection = {},
    populate = []
) => {
    const { limit, cursor, sort } = pagination;
    const sortField = Object.keys(sort)[0];
    const sortOrder = sort[sortField];
    
    // Add cursor to query if provided
    if (cursor) {
        const decodedCursor = Buffer.from(cursor, CURSOR_ENCODING).toString('utf8');
        const [cursorValue, cursorId] = decodedCursor.split('_');
        
        const cursorQuery = {
            $or: [
                {
                    [sortField]: sortOrder === 1 
                        ? { $gt: cursorValue }
                        : { $lt: cursorValue }
                },
                {
                    [sortField]: cursorValue,
                    _id: sortOrder === 1 
                        ? { $gt: cursorId }
                        : { $lt: cursorId }
                }
            ]
        };
        
        query = { $and: [query, cursorQuery] };
    }
    
    // Fetch one extra record to determine if there's a next page
    const data = await model
        .find(query, projection)
        .sort(sort)
        .limit(limit + 1)
        .populate(populate);
    
    const hasNextPage = data.length > limit;
    if (hasNextPage) {
        data.pop(); // Remove the extra record
    }
    
    // Create cursors
    const nextCursor = hasNextPage
        ? Buffer.from(
            `${data[data.length - 1][sortField]}_${data[data.length - 1]._id}`
        ).toString(CURSOR_ENCODING)
        : null;
    
    const prevCursor = cursor
        ? Buffer.from(
            `${data[0][sortField]}_${data[0]._id}`
        ).toString(CURSOR_ENCODING)
        : null;
    
    return {
        data,
        meta: {
            [META_FIELDS.pageSize]: limit,
            [META_FIELDS.hasNextPage]: hasNextPage,
            [META_FIELDS.hasPrevPage]: !!cursor,
            [META_FIELDS.nextCursor]: nextCursor,
            [META_FIELDS.prevCursor]: prevCursor
        }
    };
};

// Helper for generating pagination links
const generatePaginationLinks = (req, meta) => {
    const { protocol, hostname, originalUrl } = req;
    const baseUrl = `${protocol}://${hostname}`;
    const queryParams = new URLSearchParams(req.query);
    
    const links = {};
    
    if (meta.hasNextPage) {
        queryParams.set('page', meta.currentPage + 1);
        links.next = `${baseUrl}${originalUrl.split('?')[0]}?${queryParams}`;
    }
    
    if (meta.hasPrevPage) {
        queryParams.set('page', meta.currentPage - 1);
        links.prev = `${baseUrl}${originalUrl.split('?')[0]}?${queryParams}`;
    }
    
    return links;
};

module.exports = {
    createOffsetPaginationResponse,
    createCursorPaginationResponse,
    generatePaginationLinks
};

Updated Book Routes

Update src/routes/books.js to implement pagination:

const express = require('express');
const router = express.Router();
const { Book } = require('../models/book');
const { offsetPagination, cursorPagination } = require('../middleware/pagination');
const { 
    createOffsetPaginationResponse,
    createCursorPaginationResponse,
    generatePaginationLinks 
} = require('../utils/paginationHelpers');

// Get all books with offset-based pagination
router.get('/', offsetPagination, async (req, res) => {
    try {
        const query = {};
        
        // Apply filters
        if (req.query.genre) {
            query.genre = req.query.genre;
        }
        if (req.query.author) {
            query.author = new RegExp(req.query.author, 'i');
        }
        
        const result = await createOffsetPaginationResponse(
            Book,
            query,
            req.pagination,
            {}, // Projection
            ['author'] // Populate
        );
        
        // Generate HATEOAS links
        const links = generatePaginationLinks(req, result.meta);
        
        res.json({
            ...result,
            links
        });
    } catch (error) {
        res.status(500).json({ error: error.message });
    }
});

// Get all books with cursor-based pagination
router.get('/cursor', cursorPagination, async (req, res) => {
    try {
        const query = {};
        
        // Apply filters
        if (req.query.genre) {
            query.genre = req.query.genre;
        }
        if (req.query.author) {
            query.author = new RegExp(req.query.author, 'i');
        }
        
        const result = await createCursorPaginationResponse(
            Book,
            query,
            req.pagination,
            {}, // Projection
            ['author'] // Populate
        );
        
        res.json(result);
    } catch (error) {
        res.status(500).json({ error: error.message });
    }
});

Response Headers

Add pagination metadata to response headers:

const addPaginationHeaders = (req, res, next) => {
    const sendResponse = res.json;
    res.json = function(body) {
        if (body && body.meta) {
            res.set({
                'X-Total-Count': body.meta.totalCount,
                'X-Page-Size': body.meta.pageSize,
                'X-Current-Page': body.meta.currentPage,
                'X-Total-Pages': body.meta.totalPages,
                'X-Has-Next-Page': body.meta.hasNextPage,
                'X-Has-Prev-Page': body.meta.hasPrevPage
            });
        }
        return sendResponse.call(this, body);
    };
    next();
};

Pagination Tests

Create __tests__/integration/pagination.test.js:

const request = require('supertest');
const app = require('../../src/server');
const { createTestBook } = require('../../src/utils/testUtils');

describe('Pagination Tests', () => {
    // Create test data
    beforeEach(async () => {
        const books = Array(15).fill().map(() => createTestBook());
        await Promise.all(books);
    });
    
    describe('Offset-based Pagination', () => {
        it('should return paginated results with default settings', async () => {
            const response = await request(app)
                .get('/api/books')
                .expect(200);
            
            expect(response.body.data).toHaveLength(10); // Default page size
            expect(response.body.meta.currentPage).toBe(1);
            expect(response.body.meta.hasNextPage).toBe(true);
        });
        
        it('should respect custom page size', async () => {
            const response = await request(app)
                .get('/api/books?limit=5')
                .expect(200);
            
            expect(response.body.data).toHaveLength(5);
        });
        
        it('should enforce maximum page size', async () => {
            const response = await request(app)
                .get('/api/books?limit=200')
                .expect(200);
            
            expect(response.body.data.length).toBeLessThanOrEqual(100);
        });
    });
    
    describe('Cursor-based Pagination', () => {
        it('should return cursor-based results', async () => {
            const response = await request(app)
                .get('/api/books/cursor')
                .expect(200);
            
            expect(response.body.data).toBeDefined();
            expect(response.body.meta.nextCursor).toBeDefined();
        });
        
        it('should handle cursor navigation', async () => {
            // Get first page
            const firstResponse = await request(app)
                .get('/api/books/cursor?limit=5')
                .expect(200);
            
            // Get next page using cursor
            const nextResponse = await request(app)
                .get(`/api/books/cursor?cursor=${firstResponse.body.meta.nextCursor}`)
                .expect(200);
            
            expect(nextResponse.body.data).toHaveLength(5);
            expect(nextResponse.body.meta.hasPrevPage).toBe(true);
        });
    });
});

Best Practices Implemented

  1. Multiple Strategies

    • Offset-based pagination
    • Cursor-based pagination
    • Flexible configuration
    • HATEOAS compliance

  2. Performance Optimization

    • Parallel query execution
    • Cursor-based efficiency
    • Index utilization
    • Resource limiting

  3. Developer Experience

    • Clear documentation
    • Consistent responses
    • Error handling
    • Flexible configuration

  4. Client Integration

    • Response headers
    • HATEOAS links
    • Clear metadata
    • Cursor encoding

Next Steps

To enhance your API further:

  1. Implementing proper logging

Conclusion

You now have a robust pagination system that:

  • Handles large datasets efficiently
  • Provides multiple pagination strategies
  • Maintains consistent performance
  • Follows REST best practices
  • Supports various use cases

Remember to:

  • Monitor pagination performance
  • Index relevant fields
  • Update documentation
  • Test edge cases
  • Consider caching strategies

The next tutorial will cover implementing proper logging for your API endpoints.

Related Posts

API Management Tools: A Comprehensive Overview
APIs tutorialcreate an APIAPI development guideTutorial

• 5 min read

APIs (Application Programming Interfaces) are the backbone of modern digital applications. They allow different software systems to communicate, exchange data, and collaborate seamlessly. As businesse...

Read More
API Security: Best Practices for Protecting Your Application
APIs tutorialcreate an APIAPI development guideTutorial

• 4 min read

In today’s interconnected digital world, APIs (Application Programming Interfaces) are the backbone of communication between different software applications. From mobile apps to cloud services, APIs e...

Read More
API Design Best Practices: Crafting Robust and Scalable APIs
APIs tutorialcreate an APIAPI development guideTutorial

• 5 min read

In the modern digital ecosystem, APIs (Application Programming Interfaces) serve as the backbone of connectivity. Whether you're building microservices, enabling integrations, or crafting data pipelin...

Read More