AWS S3 API 的默认 getObject() 函数有一个 5GB 的大小限制,如果需要下载更大的文件,则需要使用 Multipart Downloads。以下是一个示例代码,可以实现文件的分块下载:
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
const bucketName = 'your-bucket-name';
const objectKey = 'your-object-key';
const downloadFilePath = 'download-path';
const multipartThreshold = 1024 * 1024 * 5; // 5 MB
const multipartSize = 1024 * 1024 * 5; // 5 MB
const params = {
Bucket: bucketName,
Key: objectKey,
};
s3.headObject(params, function(err, metadata) {
if (err) {
console.error(err);
return;
}
const fileSize = metadata.ContentLength;
console.log(`File size is ${fileSize}`);
if (fileSize > multipartThreshold) {
const parts = Math.ceil(fileSize / multipartSize);
console.log(`File will be downloaded in ${parts} parts`);
const downloadParams = {
...params,
SSECustomerAlgorithm: 'AES256',
SSECustomerKey: 'your-customer-key',
SSECustomerKeyMD5: 'your-customer-key-md5',
PartSize: multipartSize,
QueueSize: 4,
Key: objectKey,
Bucket: bucketName,
};
const downloader = s3.download(downloadParams);
downloader.on('error', function(err) {
console.error(`Download failed: ${err}`);
return;
});
downloader.on('end', function() {
console.log('Download completed');
});
downloader.pipe(fs.createWriteStream(downloadFilePath));
} else {
const downloadParams = {
...params,
SSECustomerAlgorithm: 'AES256',
SSECustomerKey: 'your-customer-key',
SSECustomerKeyMD5: 'your-customer-key-md5',
};
const file = fs.createWriteStream(downloadFilePath);
const downloader = s3.getObject(downloadParams).createReadStream();
downloader.on('error', function(err) {
console.error(`Download failed: ${err}`);
return;
});
downloader.on('end