Back to Blog Posts
Building Scalable REST APIs with Node.js: Architecture and Best Practices
•14 min read
Node.jsAPIBackendJavaScriptArchitecture
Introduction
Building scalable REST APIs requires more than just handling HTTP requests. This guide covers architectural patterns, best practices, and real-world techniques for creating robust Node.js APIs that can handle production workloads.
Project Structure
A well-organized project structure is crucial for maintainability:
src/
├── api/
│ ├── controllers/
│ ├── routes/
│ ├── middlewares/
│ └── validators/
├── config/
├── models/
├── services/
├── utils/
├── database/
│ ├── migrations/
│ └── seeds/
├── tests/
│ ├── unit/
│ ├── integration/
│ └── fixtures/
└── app.js
Setting Up the Foundation
Basic Express Setup with TypeScript
// src/app.ts
import express, { Application, Request, Response, NextFunction } from 'express';
import cors from 'cors';
import helmet from 'helmet';
import compression from 'compression';
import rateLimit from 'express-rate-limit';
import { errorHandler } from './middlewares/errorHandler';
import { logger } from './utils/logger';
import routes from './api/routes';
class App {
public app: Application;
constructor() {
this.app = express();
this.initializeMiddlewares();
this.initializeRoutes();
this.initializeErrorHandling();
}
private initializeMiddlewares(): void {
// Security headers
this.app.use(helmet());
// CORS configuration
this.app.use(cors({
origin: process.env.ALLOWED_ORIGINS?.split(',') || '*',
credentials: true
}));
// Body parsing
this.app.use(express.json({ limit: '10mb' }));
this.app.use(express.urlencoded({ extended: true, limit: '10mb' }));
// Compression
this.app.use(compression());
// Rate limiting
const limiter = rateLimit({
windowMs: 15 * 60 * 1000, // 15 minutes
max: 100, // limit each IP to 100 requests per windowMs
message: 'Too many requests from this IP'
});
this.app.use('/api', limiter);
// Request logging
this.app.use((req: Request, res: Response, next: NextFunction) => {
logger.info(`${req.method} ${req.path}`, {
ip: req.ip,
userAgent: req.get('user-agent')
});
next();
});
}
private initializeRoutes(): void {
this.app.use('/api/v1', routes);
// Health check
this.app.get('/health', (req: Request, res: Response) => {
res.status(200).json({
status: 'OK',
timestamp: new Date().toISOString(),
uptime: process.uptime()
});
});
}
private initializeErrorHandling(): void {
// 404 handler
this.app.use((req: Request, res: Response) => {
res.status(404).json({
error: 'Not Found',
message: `Route ${req.path} not found`
});
});
// Global error handler
this.app.use(errorHandler);
}
}
export default new App().app;
Database Layer
Database Connection with Connection Pooling
// src/database/connection.ts
import { Pool } from 'pg';
import { logger } from '../utils/logger';
class Database {
private pool: Pool;
private static instance: Database;
private constructor() {
this.pool = new Pool({
host: process.env.DB_HOST,
port: parseInt(process.env.DB_PORT || '5432'),
database: process.env.DB_NAME,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
max: 20, // Maximum number of clients in the pool
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 2000,
});
this.pool.on('error', (err) => {
logger.error('Unexpected database error', err);
});
}
public static getInstance(): Database {
if (!Database.instance) {
Database.instance = new Database();
}
return Database.instance;
}
public async query(text: string, params?: any[]): Promise<any> {
const start = Date.now();
try {
const res = await this.pool.query(text, params);
const duration = Date.now() - start;
logger.debug('Executed query', { text, duration, rows: res.rowCount });
return res;
} catch (error) {
logger.error('Database query error', { error, text });
throw error;
}
}
public async getClient() {
return await this.pool.connect();
}
public async transaction<T>(
callback: (client: any) => Promise<T>
): Promise<T> {
const client = await this.getClient();
try {
await client.query('BEGIN');
const result = await callback(client);
await client.query('COMMIT');
return result;
} catch (error) {
await client.query('ROLLBACK');
throw error;
} finally {
client.release();
}
}
}
export const db = Database.getInstance();
Repository Pattern
// src/repositories/BaseRepository.ts
export abstract class BaseRepository<T> {
protected tableName: string;
constructor(tableName: string) {
this.tableName = tableName;
}
async findAll(
filters?: Partial<T>,
options?: QueryOptions
): Promise<T[]> {
let query = `SELECT * FROM ${this.tableName}`;
const values: any[] = [];
if (filters && Object.keys(filters).length > 0) {
const conditions = Object.keys(filters).map((key, index) => {
values.push(filters[key as keyof T]);
return `${key} = $${index + 1}`;
});
query += ` WHERE ${conditions.join(' AND ')}`;
}
if (options?.orderBy) {
query += ` ORDER BY ${options.orderBy} ${options.order || 'ASC'}`;
}
if (options?.limit) {
query += ` LIMIT ${options.limit}`;
}
if (options?.offset) {
query += ` OFFSET ${options.offset}`;
}
const result = await db.query(query, values);
return result.rows;
}
async findById(id: string | number): Promise<T | null> {
const result = await db.query(
`SELECT * FROM ${this.tableName} WHERE id = $1`,
[id]
);
return result.rows[0] || null;
}
async create(data: Partial<T>): Promise<T> {
const keys = Object.keys(data);
const values = Object.values(data);
const placeholders = keys.map((_, index) => `$${index + 1}`);
const query = `
INSERT INTO ${this.tableName} (${keys.join(', ')})
VALUES (${placeholders.join(', ')})
RETURNING *
`;
const result = await db.query(query, values);
return result.rows[0];
}
async update(id: string | number, data: Partial<T>): Promise<T | null> {
const keys = Object.keys(data);
const values = Object.values(data);
const setClause = keys.map(
(key, index) => `${key} = $${index + 2}`
).join(', ');
const query = `
UPDATE ${this.tableName}
SET ${setClause}, updated_at = CURRENT_TIMESTAMP
WHERE id = $1
RETURNING *
`;
const result = await db.query(query, [id, ...values]);
return result.rows[0] || null;
}
async delete(id: string | number): Promise<boolean> {
const result = await db.query(
`DELETE FROM ${this.tableName} WHERE id = $1`,
[id]
);
return result.rowCount > 0;
}
}
// src/repositories/UserRepository.ts
interface User {
id: number;
email: string;
name: string;
password: string;
created_at: Date;
updated_at: Date;
}
export class UserRepository extends BaseRepository<User> {
constructor() {
super('users');
}
async findByEmail(email: string): Promise<User | null> {
const result = await db.query(
'SELECT * FROM users WHERE email = $1',
[email]
);
return result.rows[0] || null;
}
async updateLastLogin(userId: number): Promise<void> {
await db.query(
'UPDATE users SET last_login = CURRENT_TIMESTAMP WHERE id = $1',
[userId]
);
}
}
Authentication and Authorization
JWT Authentication Middleware
// src/middlewares/auth.ts
import jwt from 'jsonwebtoken';
import { Request, Response, NextFunction } from 'express';
import { UserRepository } from '../repositories/UserRepository';
interface AuthRequest extends Request {
user?: any;
token?: string;
}
export class AuthMiddleware {
private static userRepo = new UserRepository();
static async authenticate(
req: AuthRequest,
res: Response,
next: NextFunction
): Promise<void> {
try {
const token = req.header('Authorization')?.replace('Bearer ', '');
if (!token) {
throw new Error('No token provided');
}
const decoded = jwt.verify(
token,
process.env.JWT_SECRET!
) as { userId: number };
const user = await AuthMiddleware.userRepo.findById(decoded.userId);
if (!user) {
throw new Error('User not found');
}
req.user = user;
req.token = token;
next();
} catch (error) {
res.status(401).json({ error: 'Please authenticate' });
}
}
static authorize(...roles: string[]) {
return (req: AuthRequest, res: Response, next: NextFunction) => {
if (!req.user) {
return res.status(401).json({ error: 'Unauthorized' });
}
if (roles.length && !roles.includes(req.user.role)) {
return res.status(403).json({ error: 'Forbidden' });
}
next();
};
}
}
// src/services/AuthService.ts
export class AuthService {
private userRepo = new UserRepository();
async register(userData: RegisterDTO): Promise<AuthResponse> {
// Validate user data
const existingUser = await this.userRepo.findByEmail(userData.email);
if (existingUser) {
throw new Error('Email already registered');
}
// Hash password
const hashedPassword = await bcrypt.hash(userData.password, 10);
// Create user
const user = await this.userRepo.create({
...userData,
password: hashedPassword
});
// Generate token
const token = this.generateToken(user.id);
return {
user: this.sanitizeUser(user),
token
};
}
async login(credentials: LoginDTO): Promise<AuthResponse> {
const user = await this.userRepo.findByEmail(credentials.email);
if (!user) {
throw new Error('Invalid credentials');
}
const isPasswordValid = await bcrypt.compare(
credentials.password,
user.password
);
if (!isPasswordValid) {
throw new Error('Invalid credentials');
}
await this.userRepo.updateLastLogin(user.id);
const token = this.generateToken(user.id);
return {
user: this.sanitizeUser(user),
token
};
}
private generateToken(userId: number): string {
return jwt.sign(
{ userId },
process.env.JWT_SECRET!,
{ expiresIn: '7d' }
);
}
private sanitizeUser(user: User): Partial<User> {
const { password, ...sanitized } = user;
return sanitized;
}
}
Request Validation
Using Joi for Validation
// src/validators/schemas.ts
import Joi from 'joi';
export const schemas = {
user: {
create: Joi.object({
email: Joi.string().email().required(),
password: Joi.string().min(8).required(),
name: Joi.string().min(2).max(100).required(),
role: Joi.string().valid('user', 'admin').default('user')
}),
update: Joi.object({
email: Joi.string().email(),
name: Joi.string().min(2).max(100),
role: Joi.string().valid('user', 'admin')
}).min(1),
query: Joi.object({
page: Joi.number().integer().min(1).default(1),
limit: Joi.number().integer().min(1).max(100).default(20),
sort: Joi.string().valid('name', 'email', 'created_at'),
order: Joi.string().valid('asc', 'desc').default('asc'),
search: Joi.string().max(100)
})
}
};
// src/middlewares/validate.ts
export const validate = (schema: Joi.Schema) => {
return (req: Request, res: Response, next: NextFunction) => {
const { error, value } = schema.validate(req.body, {
abortEarly: false,
stripUnknown: true
});
if (error) {
const errors = error.details.map(detail => ({
field: detail.path.join('.'),
message: detail.message
}));
return res.status(400).json({
error: 'Validation failed',
details: errors
});
}
req.body = value;
next();
};
};
Error Handling
Centralized Error Handling
// src/utils/AppError.ts
export class AppError extends Error {
constructor(
public message: string,
public statusCode: number = 500,
public isOperational: boolean = true,
public code?: string
) {
super(message);
Object.setPrototypeOf(this, AppError.prototype);
Error.captureStackTrace(this, this.constructor);
}
}
// src/middlewares/errorHandler.ts
export const errorHandler = (
err: Error,
req: Request,
res: Response,
next: NextFunction
) => {
let error = { ...err };
error.message = err.message;
// Log error
logger.error(err);
// Mongoose bad ObjectId
if (err.name === 'CastError') {
const message = 'Resource not found';
error = new AppError(message, 404);
}
// Mongoose duplicate key
if ((err as any).code === 11000) {
const message = 'Duplicate field value entered';
error = new AppError(message, 400);
}
// Mongoose validation error
if (err.name === 'ValidationError') {
const message = Object.values((err as any).errors)
.map((val: any) => val.message)
.join(', ');
error = new AppError(message, 400);
}
res.status((error as AppError).statusCode || 500).json({
success: false,
error: {
message: error.message || 'Server Error',
...(process.env.NODE_ENV === 'development' && {
stack: err.stack,
details: err
})
}
});
};
// src/utils/catchAsync.ts
export const catchAsync = (
fn: (req: Request, res: Response, next: NextFunction) => Promise<any>
) => {
return (req: Request, res: Response, next: NextFunction) => {
Promise.resolve(fn(req, res, next)).catch(next);
};
};
API Versioning
// src/api/v1/routes/index.ts
import { Router } from 'express';
import userRoutes from './userRoutes';
import authRoutes from './authRoutes';
import productRoutes from './productRoutes';
const router = Router();
router.use('/auth', authRoutes);
router.use('/users', userRoutes);
router.use('/products', productRoutes);
export default router;
// src/api/v2/routes/index.ts
// Version 2 with breaking changes
import { Router } from 'express';
import userRoutesV2 from './userRoutes';
const router = Router();
router.use('/users', userRoutesV2); // New user structure
export default router;
Caching Strategies
Redis Caching
// src/services/CacheService.ts
import Redis from 'ioredis';
import { logger } from '../utils/logger';
export class CacheService {
private redis: Redis;
private defaultTTL: number = 3600; // 1 hour
constructor() {
this.redis = new Redis({
host: process.env.REDIS_HOST || 'localhost',
port: parseInt(process.env.REDIS_PORT || '6379'),
password: process.env.REDIS_PASSWORD,
retryStrategy: (times) => {
const delay = Math.min(times * 50, 2000);
return delay;
}
});
this.redis.on('error', (err) => {
logger.error('Redis error:', err);
});
}
async get<T>(key: string): Promise<T | null> {
try {
const value = await this.redis.get(key);
return value ? JSON.parse(value) : null;
} catch (error) {
logger.error('Cache get error:', error);
return null;
}
}
async set(
key: string,
value: any,
ttl: number = this.defaultTTL
): Promise<void> {
try {
await this.redis.set(
key,
JSON.stringify(value),
'EX',
ttl
);
} catch (error) {
logger.error('Cache set error:', error);
}
}
async invalidate(pattern: string): Promise<void> {
try {
const keys = await this.redis.keys(pattern);
if (keys.length) {
await this.redis.del(...keys);
}
} catch (error) {
logger.error('Cache invalidate error:', error);
}
}
}
// src/middlewares/cache.ts
export const cache = (keyPrefix: string, ttl?: number) => {
return async (req: Request, res: Response, next: NextFunction) => {
const key = `${keyPrefix}:${req.originalUrl}`;
const cached = await cacheService.get(key);
if (cached) {
return res.json(cached);
}
// Store original send function
const originalSend = res.json;
// Override json method
res.json = function(data: any) {
// Cache the response
cacheService.set(key, data, ttl);
// Call original send
return originalSend.call(this, data);
};
next();
};
};
Testing
Unit Testing with Jest
// src/tests/unit/services/UserService.test.ts
import { UserService } from '../../../services/UserService';
import { UserRepository } from '../../../repositories/UserRepository';
jest.mock('../../../repositories/UserRepository');
describe('UserService', () => {
let userService: UserService;
let mockUserRepo: jest.Mocked<UserRepository>;
beforeEach(() => {
mockUserRepo = new UserRepository() as jest.Mocked<UserRepository>;
userService = new UserService(mockUserRepo);
});
describe('createUser', () => {
it('should create a new user successfully', async () => {
const userData = {
email: '[email protected]',
name: 'Test User',
password: 'password123'
};
const hashedPassword = 'hashedPassword';
const createdUser = {
id: 1,
...userData,
password: hashedPassword,
created_at: new Date(),
updated_at: new Date()
};
mockUserRepo.findByEmail.mockResolvedValue(null);
mockUserRepo.create.mockResolvedValue(createdUser);
const result = await userService.createUser(userData);
expect(mockUserRepo.findByEmail).toHaveBeenCalledWith(userData.email);
expect(mockUserRepo.create).toHaveBeenCalledWith(
expect.objectContaining({
email: userData.email,
name: userData.name,
password: expect.any(String)
})
);
expect(result).not.toHaveProperty('password');
});
it('should throw error if email already exists', async () => {
const userData = {
email: '[email protected]',
name: 'Test User',
password: 'password123'
};
mockUserRepo.findByEmail.mockResolvedValue({
id: 1,
email: userData.email
} as any);
await expect(userService.createUser(userData))
.rejects
.toThrow('Email already registered');
});
});
});
Integration Testing
// src/tests/integration/auth.test.ts
import request from 'supertest';
import app from '../../app';
import { db } from '../../database/connection';
describe('Auth Endpoints', () => {
beforeAll(async () => {
// Setup test database
await db.query('DELETE FROM users');
});
afterAll(async () => {
// Clean up
await db.close();
});
describe('POST /api/v1/auth/register', () => {
it('should register a new user', async () => {
const response = await request(app)
.post('/api/v1/auth/register')
.send({
email: '[email protected]',
password: 'password123',
name: 'Test User'
});
expect(response.status).toBe(201);
expect(response.body).toHaveProperty('token');
expect(response.body.user).toHaveProperty('email', '[email protected]');
expect(response.body.user).not.toHaveProperty('password');
});
it('should not register user with existing email', async () => {
const response = await request(app)
.post('/api/v1/auth/register')
.send({
email: '[email protected]',
password: 'password123',
name: 'Another User'
});
expect(response.status).toBe(400);
expect(response.body).toHaveProperty('error');
});
});
});
Performance Optimization
Query Optimization
// src/repositories/ProductRepository.ts
export class ProductRepository extends BaseRepository<Product> {
async findWithPagination(
filters: ProductFilters,
pagination: PaginationParams
): Promise<PaginatedResult<Product>> {
const { page = 1, limit = 20 } = pagination;
const offset = (page - 1) * limit;
// Build efficient query with indexes
let query = `
SELECT p.*, c.name as category_name
FROM products p
LEFT JOIN categories c ON p.category_id = c.id
WHERE 1=1
`;
const values: any[] = [];
let paramCount = 0;
if (filters.categoryId) {
values.push(filters.categoryId);
query += ` AND p.category_id = $${++paramCount}`;
}
if (filters.minPrice) {
values.push(filters.minPrice);
query += ` AND p.price >= $${++paramCount}`;
}
if (filters.search) {
values.push(`%${filters.search}%`);
query += ` AND (p.name ILIKE $${++paramCount} OR p.description ILIKE $${paramCount})`;
}
// Get total count
const countQuery = `SELECT COUNT(*) FROM (${query}) as filtered`;
const countResult = await db.query(countQuery, values);
const total = parseInt(countResult.rows[0].count);
// Add pagination
values.push(limit, offset);
query += ` ORDER BY p.created_at DESC LIMIT $${++paramCount} OFFSET $${++paramCount}`;
const result = await db.query(query, values);
return {
data: result.rows,
pagination: {
page,
limit,
total,
pages: Math.ceil(total / limit)
}
};
}
}
Response Compression and Streaming
// src/controllers/ReportController.ts
export class ReportController {
async streamLargeReport(req: Request, res: Response) {
const { startDate, endDate } = req.query;
res.setHeader('Content-Type', 'application/json');
res.setHeader('Content-Disposition', 'attachment; filename="report.json"');
// Start streaming response
res.write('{"data":[');
let first = true;
const stream = db.queryStream(`
SELECT * FROM transactions
WHERE created_at BETWEEN $1 AND $2
ORDER BY created_at
`, [startDate, endDate]);
stream.on('data', (row) => {
if (!first) res.write(',');
res.write(JSON.stringify(row));
first = false;
});
stream.on('end', () => {
res.write(']}');
res.end();
});
stream.on('error', (error) => {
logger.error('Stream error:', error);
res.status(500).end();
});
}
}
Monitoring and Logging
// src/utils/logger.ts
import winston from 'winston';
import DailyRotateFile from 'winston-daily-rotate-file';
const logFormat = winston.format.combine(
winston.format.timestamp(),
winston.format.errors({ stack: true }),
winston.format.json()
);
export const logger = winston.createLogger({
level: process.env.LOG_LEVEL || 'info',
format: logFormat,
defaultMeta: { service: 'api' },
transports: [
// Console transport
new winston.transports.Console({
format: winston.format.combine(
winston.format.colorize(),
winston.format.simple()
)
}),
// File transport with rotation
new DailyRotateFile({
filename: 'logs/application-%DATE%.log',
datePattern: 'YYYY-MM-DD',
zippedArchive: true,
maxSize: '20m',
maxFiles: '14d'
}),
// Error log file
new winston.transports.File({
filename: 'logs/error.log',
level: 'error'
})
]
});
// src/middlewares/requestLogger.ts
export const requestLogger = (req: Request, res: Response, next: NextFunction) => {
const start = Date.now();
res.on('finish', () => {
const duration = Date.now() - start;
logger.info('HTTP Request', {
method: req.method,
url: req.originalUrl,
status: res.statusCode,
duration,
ip: req.ip,
userAgent: req.get('user-agent'),
userId: (req as any).user?.id
});
});
next();
};
Security Best Practices
// src/middlewares/security.ts
import helmet from 'helmet';
import mongoSanitize from 'express-mongo-sanitize';
import hpp from 'hpp';
export const setupSecurity = (app: Application) => {
// Helmet for security headers
app.use(helmet({
contentSecurityPolicy: {
directives: {
defaultSrc: ["'self'"],
styleSrc: ["'self'", "'unsafe-inline'"],
scriptSrc: ["'self'"],
imgSrc: ["'self'", "data:", "https:"],
},
},
}));
// Prevent NoSQL injection attacks
app.use(mongoSanitize());
// Prevent parameter pollution
app.use(hpp({
whitelist: ['sort', 'fields', 'page', 'limit']
}));
// Custom security headers
app.use((req, res, next) => {
res.setHeader('X-Content-Type-Options', 'nosniff');
res.setHeader('X-Frame-Options', 'DENY');
res.setHeader('X-XSS-Protection', '1; mode=block');
res.setHeader('Strict-Transport-Security', 'max-age=31536000; includeSubDomains');
next();
});
};
Deployment Considerations
Docker Configuration
# Dockerfile
FROM node:18-alpine AS builder
WORKDIR /app
# Copy package files
COPY package*.json ./
COPY tsconfig.json ./
# Install dependencies
RUN npm ci --only=production
# Copy source code
COPY src ./src
# Build TypeScript
RUN npm run build
# Production stage
FROM node:18-alpine
WORKDIR /app
# Copy built application
COPY /app/dist ./dist
COPY /app/node_modules ./node_modules
COPY /app/package*.json ./
# Create non-root user
RUN addgroup -g 1001 -S nodejs
RUN adduser -S nodejs -u 1001
USER nodejs
EXPOSE 3000
CMD ["node", "dist/server.js"]
Conclusion
Building scalable REST APIs with Node.js requires careful consideration of architecture, performance, security, and maintainability. Key takeaways:
- Structure matters: Organize your code for maintainability
- Layer appropriately: Separate concerns (routes, controllers, services, repositories)
- Handle errors gracefully: Implement centralized error handling
- Validate everything: Never trust user input
- Cache strategically: Use caching to improve performance
- Test thoroughly: Write both unit and integration tests
- Monitor continuously: Log and track API performance
- Secure by default: Implement security best practices from the start
Remember, the best API is one that's reliable, performant, and easy to maintain. Happy coding!