Merge pull request #737 from NapNeko/ffmpeg

feat: no spawn ffmpeg
This commit is contained in:
手瓜一十雪
2025-01-26 21:02:57 +08:00
committed by GitHub
9 changed files with 169 additions and 177 deletions

View File

@@ -30,7 +30,6 @@
"@types/cors": "^2.8.17",
"@sinclair/typebox": "^0.34.9",
"@types/express": "^5.0.0",
"@types/fluent-ffmpeg": "^2.1.24",
"@types/node": "^22.0.1",
"@types/qrcode-terminal": "^0.12.2",
"@types/ws": "^8.5.12",
@@ -55,8 +54,9 @@
"winston": "^3.17.0"
},
"dependencies": {
"@ffmpeg.wasm/core-mt": "^0.13.2",
"@ffmpeg.wasm/main": "^0.13.1",
"express": "^5.0.0",
"fluent-ffmpeg": "^2.1.2",
"piscina": "^4.7.0",
"qrcode-terminal": "^0.12.0",
"silk-wasm": "^3.6.1",

View File

@@ -2,14 +2,12 @@ import Piscina from 'piscina';
import fsPromise from 'fs/promises';
import path from 'node:path';
import { randomUUID } from 'crypto';
import { spawn } from 'node:child_process';
import { EncodeResult, getDuration, getWavFileInfo, isSilk, isWav } from 'silk-wasm';
import { LogWrapper } from '@/common/log';
import { EncodeArgs } from "@/common/audio-worker";
import { FFmpegService } from "@/common/ffmpeg";
const ALLOW_SAMPLE_RATE = [8000, 12000, 16000, 24000, 32000, 44100, 48000];
const EXIT_CODES = [0, 255];
const FFMPEG_PATH = process.env.FFMPEG_PATH ?? 'ffmpeg';
async function getWorkerPath() {
return new URL(/* @vite-ignore */ './audio-worker.mjs', import.meta.url).href;
@@ -26,30 +24,6 @@ async function guessDuration(pttPath: string, logger: LogWrapper) {
return duration;
}
async function convert(filePath: string, pcmPath: string, logger: LogWrapper): Promise<Buffer> {
return new Promise<Buffer>((resolve, reject) => {
const cp = spawn(FFMPEG_PATH, ['-y', '-i', filePath, '-ar', '24000', '-ac', '1', '-f', 's16le', pcmPath]);
cp.on('error', (err: Error) => {
logger.log('FFmpeg处理转换出错: ', err.message);
reject(err);
});
cp.on('exit', async (code, signal) => {
if (code == null || EXIT_CODES.includes(code)) {
try {
const data = await fsPromise.readFile(pcmPath);
await fsPromise.unlink(pcmPath);
resolve(data);
} catch (err) {
reject(err);
}
} else {
logger.log(`FFmpeg exit: code=${code ?? 'unknown'} sig=${signal ?? 'unknown'}`);
reject(new Error('FFmpeg处理转换失败'));
}
});
});
}
async function handleWavFile(
file: Buffer,
filePath: string,
@@ -58,7 +32,7 @@ async function handleWavFile(
): Promise<{ input: Buffer; sampleRate: number }> {
const { fmt } = getWavFileInfo(file);
if (!ALLOW_SAMPLE_RATE.includes(fmt.sampleRate)) {
return { input: await convert(filePath, pcmPath, logger), sampleRate: 24000 };
return { input: await FFmpegService.convert(filePath, pcmPath, logger), sampleRate: 24000 };
}
return { input: file, sampleRate: fmt.sampleRate };
}
@@ -72,7 +46,7 @@ export async function encodeSilk(filePath: string, TEMP_DIR: string, logger: Log
const pcmPath = `${pttPath}.pcm`;
const { input, sampleRate } = isWav(file)
? (await handleWavFile(file, filePath, pcmPath, logger))
: { input: await convert(filePath, pcmPath, logger), sampleRate: 24000 };
: { input: await FFmpegService.convert(filePath, pcmPath, logger), sampleRate: 24000 };
const silk = await piscina.run({ input: input, sampleRate: sampleRate });
await fsPromise.writeFile(pttPath, Buffer.from(silk.data));
logger.log(`语音文件${filePath}转换成功!`, pttPath, '时长:', silk.duration);

131
src/common/ffmpeg.ts Normal file
View File

@@ -0,0 +1,131 @@
import { FFmpeg } from '@ffmpeg.wasm/main';
import { randomUUID } from 'crypto';
import { readFileSync, statSync, writeFileSync } from 'fs';
import { LogWrapper } from './log';
import { VideoInfo } from './video';
import { fileTypeFromFile } from 'file-type';
import imageSize from 'image-size';
export class FFmpegService {
public static async extractThumbnail(videoPath: string, thumbnailPath: string): Promise<void> {
const ffmpegInstance = await FFmpeg.create({ core: '@ffmpeg.wasm/core-mt' });
const videoFileName = `${randomUUID()}.mp4`;
const outputFileName = `${randomUUID()}.jpg`;
try {
ffmpegInstance.fs.writeFile(videoFileName, readFileSync(videoPath));
let code = await ffmpegInstance.run('-i', videoFileName, '-ss', '00:00:01.000', '-vframes', '1', outputFileName);
if (code !== 0) {
throw new Error('Error extracting thumbnail: FFmpeg process exited with code ' + code);
}
const thumbnail = ffmpegInstance.fs.readFile(outputFileName);
writeFileSync(thumbnailPath, thumbnail);
} catch (error) {
console.error('Error extracting thumbnail:', error);
throw error;
} finally {
try {
ffmpegInstance.fs.unlink(outputFileName);
} catch (unlinkError) {
console.error('Error unlinking output file:', unlinkError);
}
try {
ffmpegInstance.fs.unlink(videoFileName);
} catch (unlinkError) {
console.error('Error unlinking video file:', unlinkError);
}
}
}
public static async convertFile(inputFile: string, outputFile: string, format: string): Promise<void> {
const ffmpegInstance = await FFmpeg.create({ core: '@ffmpeg.wasm/core-mt' });
const inputFileName = `${randomUUID()}.pcm`;
const outputFileName = `${randomUUID()}.${format}`;
try {
ffmpegInstance.fs.writeFile(inputFileName, readFileSync(inputFile));
const params = format === 'amr'
? ['-f', 's16le', '-ar', '24000', '-ac', '1', '-i', inputFileName, '-ar', '8000', '-b:a', '12.2k', outputFileName]
: ['-f', 's16le', '-ar', '24000', '-ac', '1', '-i', inputFileName, outputFileName];
let code = await ffmpegInstance.run(...params);
if (code !== 0) {
throw new Error('Error extracting thumbnail: FFmpeg process exited with code ' + code);
}
const outputData = ffmpegInstance.fs.readFile(outputFileName);
writeFileSync(outputFile, outputData);
} catch (error) {
console.error('Error converting file:', error);
throw error;
} finally {
try {
ffmpegInstance.fs.unlink(outputFileName);
} catch (unlinkError) {
console.error('Error unlinking output file:', unlinkError);
}
try {
ffmpegInstance.fs.unlink(inputFileName);
} catch (unlinkError) {
console.error('Error unlinking input file:', unlinkError);
}
}
}
public static async convert(filePath: string, pcmPath: string, logger: LogWrapper): Promise<Buffer> {
const ffmpegInstance = await FFmpeg.create({ core: '@ffmpeg.wasm/core-mt' });
const inputFileName = `${randomUUID()}.input`;
const outputFileName = `${randomUUID()}.pcm`;
try {
ffmpegInstance.fs.writeFile(inputFileName, readFileSync(filePath));
const params = ['-y', '-i', inputFileName, '-ar', '24000', '-ac', '1', '-f', 's16le', outputFileName];
let code = await ffmpegInstance.run(...params);
if (code !== 0) {
throw new Error('FFmpeg process exited with code ' + code);
}
const outputData = ffmpegInstance.fs.readFile(outputFileName);
writeFileSync(pcmPath, outputData);
return Buffer.from(outputData);
} catch (error: any) {
throw new Error('FFmpeg处理转换出错: ' + error.message);
} finally {
try {
ffmpegInstance.fs.unlink(outputFileName);
} catch (unlinkError) {
logger.log('Error unlinking output file:', unlinkError);
}
try {
ffmpegInstance.fs.unlink(inputFileName);
} catch (unlinkError) {
logger.log('Error unlinking input file:', unlinkError);
}
}
}
public static async getVideoInfo(videoPath: string, thumbnailPath: string): Promise<VideoInfo> {
await FFmpegService.extractThumbnail(videoPath, thumbnailPath);
let fileType = (await fileTypeFromFile(videoPath))?.ext ?? 'mp4';
const inputFileName = `${randomUUID()}.${fileType}`;
const ffmpegInstance = await FFmpeg.create({ core: '@ffmpeg.wasm/core-mt' });
ffmpegInstance.fs.writeFile(inputFileName, readFileSync(videoPath));
ffmpegInstance.setLogging(true);
let duration = 60;
ffmpegInstance.setLogger((level, ...msg) => {
const message = msg.join(' ');
const durationMatch = message.match(/Duration: (\d+):(\d+):(\d+\.\d+)/);
if (durationMatch) {
const hours = parseInt(durationMatch[1], 10);
const minutes = parseInt(durationMatch[2], 10);
const seconds = parseFloat(durationMatch[3]);
duration = hours * 3600 + minutes * 60 + seconds;
}
});
await ffmpegInstance.run('-i', inputFileName);
let image = imageSize(thumbnailPath);
ffmpegInstance.fs.unlink(inputFileName);
const fileSize = statSync(videoPath).size;
return {
width: image.width ?? 100,
height: image.height ?? 100,
time: duration,
format: fileType,
size: fileSize,
filePath: videoPath
}
}
}

File diff suppressed because one or more lines are too long

View File

@@ -22,11 +22,11 @@ import { ISizeCalculationResult } from 'image-size/dist/types/interface';
import { RkeyManager } from '@/core/helper/rkey';
import { calculateFileMD5 } from '@/common/file';
import pathLib from 'node:path';
import { defaultVideoThumbB64, getVideoInfo } from '@/common/video';
import ffmpeg from 'fluent-ffmpeg';
import { defaultVideoThumbB64 } from '@/common/video';
import { encodeSilk } from '@/common/audio';
import { SendMessageContext } from '@/onebot/api';
import { getFileTypeForSendType } from '../helper/msg';
import { FFmpegService } from '@/common/ffmpeg';
export class NTQQFileApi {
context: InstanceContext;
@@ -149,12 +149,6 @@ export class NTQQFileApi {
size: 0,
filePath,
};
try {
videoInfo = await getVideoInfo(filePath, this.context.logger);
} catch (e) {
this.context.logger.logError('获取视频信息失败,将使用默认值', e);
}
let fileExt = 'mp4';
try {
const tempExt = (await fileTypeFromFile(filePath))?.ext;
@@ -162,53 +156,29 @@ export class NTQQFileApi {
} catch (e) {
this.context.logger.logError('获取文件类型失败', e);
}
const newFilePath = filePath + '.' + fileExt;
const newFilePath = `${filePath}.${fileExt}`;
fs.copyFileSync(filePath, newFilePath);
context.deleteAfterSentFiles.push(newFilePath);
filePath = newFilePath;
const { fileName: _fileName, path, fileSize, md5 } = await this.core.apis.FileApi.uploadFile(filePath, ElementType.VIDEO);
if (fileSize === 0) {
throw new Error('文件异常大小为0');
}
videoInfo.size = fileSize;
let thumb = path.replace(`${pathLib.sep}Ori${pathLib.sep}`, `${pathLib.sep}Thumb${pathLib.sep}`);
thumb = pathLib.dirname(thumb);
const thumbPath = new Map();
const _thumbPath = await new Promise<string | undefined>((resolve, reject) => {
const thumbFileName = `${md5}_0.png`;
const thumbPath = pathLib.join(thumb, thumbFileName);
ffmpeg(filePath)
.on('error', (err) => {
const thumbDir = path.replace(`${pathLib.sep}Ori${pathLib.sep}`, `${pathLib.sep}Thumb${pathLib.sep}`);
fs.mkdirSync(pathLib.dirname(thumbDir), { recursive: true });
const thumbPath = pathLib.join(pathLib.dirname(thumbDir), `${md5}_0.png`);
try {
this.context.logger.logDebug('获取视频封面失败,使用默认封面', err);
if (diyThumbPath) {
fsPromises.copyFile(diyThumbPath, thumbPath).then(() => {
resolve(thumbPath);
}).catch(reject);
} else {
fs.writeFileSync(thumbPath, Buffer.from(defaultVideoThumbB64, 'base64'));
resolve(thumbPath);
}
videoInfo = await FFmpegService.getVideoInfo(filePath, thumbPath);
} catch (error) {
this.context.logger.logError('获取视频封面失败,使用默认封面失败', error);
fs.writeFileSync(thumbPath, Buffer.from(defaultVideoThumbB64, 'base64'));
}
})
.screenshots({
timestamps: [0],
filename: thumbFileName,
folder: thumb,
size: videoInfo.width + 'x' + videoInfo.height,
})
.on('end', () => {
resolve(thumbPath);
});
});
const thumbSize = _thumbPath ? (await fsPromises.stat(_thumbPath)).size : 0;
thumbPath.set(0, _thumbPath);
const thumbMd5 = _thumbPath ? await calculateFileMD5(_thumbPath) : '';
const thumbSize = (await fsPromises.stat(thumbPath)).size;
const thumbMd5 = await calculateFileMD5(thumbPath);
context.deleteAfterSentFiles.push(path);
const uploadName = (fileName || _fileName).toLocaleLowerCase().endsWith('.' + fileExt.toLocaleLowerCase()) ? (fileName || _fileName) : (fileName || _fileName) + '.' + fileExt;
const uploadName = (fileName || _fileName).toLocaleLowerCase().endsWith(`.${fileExt.toLocaleLowerCase()}`) ? (fileName || _fileName) : `${fileName || _fileName}.${fileExt}`;
return {
elementType: ElementType.VIDEO,
elementId: '',
@@ -218,15 +188,14 @@ export class NTQQFileApi {
videoMd5: md5,
thumbMd5,
fileTime: videoInfo.time,
thumbPath: thumbPath,
thumbPath: new Map([[0, thumbPath]]),
thumbSize,
thumbWidth: videoInfo.width,
thumbHeight: videoInfo.height,
fileSize: '' + fileSize,
fileSize: fileSize.toString(),
},
};
}
async createValidSendPttElement(pttPath: string): Promise<SendPttElement> {
const { converted, path: silkPath, duration } = await encodeSilk(pttPath, this.core.NapCatTempPath, this.core.context.logger);

View File

@@ -1,9 +1,8 @@
import { GetFileBase, GetFilePayload, GetFileResponse } from './GetFile';
import { ActionName } from '@/onebot/action/router';
import { spawn } from 'node:child_process';
import { promises as fs } from 'fs';
import { decode } from 'silk-wasm';
const FFMPEG_PATH = process.env.FFMPEG_PATH || 'ffmpeg';
import { FFmpegService } from '@/common/ffmpeg';
const out_format = ['mp3' , 'amr' , 'wma' , 'm4a' , 'spx' , 'ogg' , 'wav' , 'flac'];
@@ -30,7 +29,7 @@ export default class GetRecord extends GetFileBase {
await fs.access(outputFile);
} catch (error) {
await this.decodeFile(inputFile, pcmFile);
await this.convertFile(pcmFile, outputFile, payload.out_format);
await FFmpegService.convertFile(pcmFile, outputFile, payload.out_format);
}
const base64Data = await fs.readFile(outputFile, { encoding: 'base64' });
res.file = outputFile;
@@ -54,23 +53,4 @@ export default class GetRecord extends GetFileBase {
throw error; // 重新抛出错误以便调用者可以处理
}
}
private convertFile(inputFile: string, outputFile: string, format: string): Promise<void> {
return new Promise((resolve, reject) => {
const params = format === 'amr' ? ['-f', 's16le', '-ar', '24000', '-ac', '1', '-i', inputFile, '-ar', '8000', '-b:a', '12.2k', outputFile] : ['-f', 's16le', '-ar', '24000', '-ac', '1', '-i', inputFile, outputFile];
const ffmpeg = spawn(FFMPEG_PATH, params);
ffmpeg.on('close', (code) => {
if (code === 0) {
resolve();
} else {
reject(new Error(`ffmpeg process exited with code ${code}`));
}
});
ffmpeg.on('error', (error: Error) => {
reject(error);
});
});
}
}

View File

@@ -29,6 +29,7 @@ import { InitWebUi } from '@/webui';
import { WebUiDataRuntime } from '@/webui/src/helper/Data';
import { napCatVersion } from '@/common/version';
import { NodeIO3MiscListener } from '@/core/listeners/NodeIO3MiscListener';
import { FFmpegService } from '@/common/ffmpeg';
// NapCat Shell App ES 入口文件
async function handleUncaughtExceptions(logger: LogWrapper) {
process.on('uncaughtException', (err) => {

View File

@@ -4,7 +4,7 @@ import { resolve } from 'path';
import nodeResolve from '@rollup/plugin-node-resolve';
import { builtinModules } from 'module';
//依赖排除
const external = ['silk-wasm', 'ws', 'express', 'qrcode-terminal', 'fluent-ffmpeg', 'piscina'];
const external = ['silk-wasm', 'ws', 'express', 'qrcode-terminal', 'piscina', '@ffmpeg.wasm/core-mt', "@ffmpeg.wasm/main"];
const nodeModules = [...builtinModules, builtinModules.map((m) => `node:${m}`)].flat();
let startScripts: string[] | undefined = undefined;
@@ -79,7 +79,6 @@ const UniversalBaseConfig = () =>
alias: {
'@/core': resolve(__dirname, './src/core'),
'@': resolve(__dirname, './src'),
'./lib-cov/fluent-ffmpeg': './lib/fluent-ffmpeg',
'@webapi': resolve(__dirname, './src/webui/src'),
},
},
@@ -109,7 +108,6 @@ const ShellBaseConfig = () =>
alias: {
'@/core': resolve(__dirname, './src/core'),
'@': resolve(__dirname, './src'),
'./lib-cov/fluent-ffmpeg': './lib/fluent-ffmpeg',
'@webapi': resolve(__dirname, './src/webui/src'),
},
},
@@ -138,7 +136,6 @@ const FrameworkBaseConfig = () =>
alias: {
'@/core': resolve(__dirname, './src/core'),
'@': resolve(__dirname, './src'),
'./lib-cov/fluent-ffmpeg': './lib/fluent-ffmpeg',
'@webapi': resolve(__dirname, './src/webui/src'),
},
},