Uploading large files in web applications can be challenging, especially when dealing with limitations like body size limits. In this post, we’ll explore how to efficiently handle large file uploads in a Next.js application using chunking and Server Actions. We’ll break the file into smaller chunks, upload them one by one, merge them on the server, and delete the temporary chunks after the merge is complete.
Introduction
When working with file uploads, many frameworks (including Next.js) impose a body size limit on requests. In Next.js 14, for instance, the default body size limit for requests in Server Actions is 1MB. This can be a problem when uploading large files, so a common solution is to split the file into smaller chunks and upload them individually.
Step 1: Chunking the File on the Client Side
We begin by splitting the file into smaller chunks (1MB each in this case) on the client side and uploading each chunk using the uploadImage Server Action.
<!-- Frontend: Chunking and Uploading -->
<script type="text/jsx">
"use client";
import { uploadImage } from '@/action/upload-image';
import React, { ChangeEvent, FormEvent, useState } from 'react';
import { useFormState } from 'react-dom';
const Home = () => {
const [state, action] = useFormState(uploadImage, {
progress: 0,
});
const [file, setFile] = useState<File>();
const [isUploading, setIsUploading] = useState(false);
// Handles the file selection event
const handleFileChange = (e: ChangeEvent<HTMLInputElement>) => {
const files = e.target.files;
if (files?.length) {
setFile(files[0]);
}
};
// Uploads each chunk to the server using the Server Action
const uploadChunk = async (
file: File,
start: number,
end: number,
chunkNumber: number,
totalChunks: number,
) => {
const formData = new FormData();
formData.append('file', file.slice(start, end)); // Slice the file into chunks
formData.append('totalChunks', totalChunks.toString()); // Total number of chunks
formData.append('chunkNumber', chunkNumber.toString()); // Current chunk number
formData.append('fileName', file.name); // File name for the merged file
formData.append('fileType', file.type); // File type for proper handling on the server
// Call the server action with the form data
action(formData);
};
// Handles the submit event and starts the chunk upload process
const handleSubmit = async (e: FormEvent<HTMLButtonElement>) => {
e.preventDefault();
if (!file) return; // Check if file is selected
setIsUploading(true);
const chunkSize = 1024 * 1024; // 1MB per chunk
const totalChunks = Math.ceil(file.size / chunkSize);
try {
// Loop through all chunks and upload each one
for (let i = 0; i < totalChunks; i++) {
const start = i * chunkSize;
const end = Math.min((i + 1) * chunkSize, file.size);
await uploadChunk(file, start, end, i + 1, totalChunks);
}
} catch (error) {
console.error("Error during upload:", error);
} finally {
setIsUploading(false); // Set uploading to false when done
}
};
return (
<div>
<input type="file" name="file" onChange={handleFileChange} />
<button onClick={handleSubmit}>Upload {state.progress}%</button>
</div>
);
};
export default Home;
</script>Step 2: Handling File Uploads with Server Actions
On the server side, we use the uploadImage Server Action to handle each chunk. When all chunks have been uploaded, the server merges them into a single file and deletes the temporary chunk files.
<!-- Backend: Handling Chunks on the Server -->
<script type="text/typescript">
'use server';
import fs from 'fs';
import path from 'path';
const UPLOAD_DIR = path.join(process.cwd(), 'uploads');
// Ensure the uploads directory exists
if (!fs.existsSync(UPLOAD_DIR)) {
fs.mkdirSync(UPLOAD_DIR);
}
// Server Action to handle the file chunk upload
export const uploadImage = async (_: any, formData: FormData) => {
const file = formData.get('file') as Blob;
const totalChunks = formData.get('totalChunks') as string;
const chunkNumber = formData.get('chunkNumber') as string;
const fileName = formData.get('fileName') as string;
const chunkPath = path.join(UPLOAD_DIR, 'chunk-${chunkNumber}');
const buffer = Buffer.from(await file.arrayBuffer());
// Write the chunk to the server
fs.writeFileSync(chunkPath, buffer);
// Check if the last chunk is uploaded and merge the file
if (totalChunks === chunkNumber) {
const mergedFilePath = path.join(UPLOAD_DIR, '${fileName}');
const writeStream = fs.createWriteStream(mergedFilePath);
// Merge the chunks
for (let i = 1; i <= parseInt(totalChunks); i++) {
const chunk = fs.readFileSync(path.join(UPLOAD_DIR, 'chunk-${i}'));
writeStream.write(chunk);
}
writeStream.end();
// After merging, remove the chunk files
for (let i = 1; i <= parseInt(totalChunks); i++) {
const chunkFilePath = path.join(UPLOAD_DIR, 'chunk-${i}');
fs.unlinkSync(chunkFilePath); // Synchronously delete the chunk file
}
}
// Return the upload progress
return {
progress: Math.min(((+chunkNumber + 1) / +totalChunks) * 100, 100),
};
};
</script>Step 3: Cleanup After Merging the File
Once the chunks are merged, we need to clean up the temporary chunk files. This is done by calling fs.unlinkSync() to delete each chunk after it has been written to the final file.
In the uploadImage server action, after merging all chunks, we loop through all the chunk files and delete them:
for (let i = 1; i <= parseInt(totalChunks); i++) {
const chunkFilePath = path.join(UPLOAD_DIR, 'chunk-${i}');
fs.unlinkSync(chunkFilePath); // Synchronously delete the chunk file
}Conclusion
By chunking large files into smaller pieces, we can bypass the body size limits imposed by Next.js and other frameworks. The Server Action makes it easier to handle file uploads on the server side, and the cleanup process ensures that temporary chunk files are deleted after use.
This approach is perfect for handling large file uploads in Next.js apps, and it can be adapted to work with different file types, chunk sizes, and other upload requirements.
Happy coding!