From 6ab82739a64000702e01d52515e2f709bf8b35cb Mon Sep 17 00:00:00 2001 From: pk5ls20 Date: Sat, 2 Nov 2024 01:51:57 +0800 Subject: [PATCH] feat: ai voice --- src/core/apis/packet.ts | 51 ++++++++++++++++- src/core/packet/entities/aiChat.ts | 16 ++++++ src/core/packet/packer.ts | 60 +++++++++++++++++++- src/core/packet/proto/oidb/Oidb.0x929.ts | 42 ++++++++++++++ src/core/packet/proto/oidb/OidbBase.ts | 3 +- src/onebot/action/extends/GetAiCharacters.ts | 41 +++++++++++++ src/onebot/action/group/GetAiRecord.ts | 28 +++++++++ src/onebot/action/group/SendGroupAiRecord.ts | 40 +++++++++++++ src/onebot/action/index.ts | 6 ++ src/onebot/action/types.ts | 3 + 10 files changed, 286 insertions(+), 4 deletions(-) create mode 100644 src/core/packet/entities/aiChat.ts create mode 100644 src/core/packet/proto/oidb/Oidb.0x929.ts create mode 100644 src/onebot/action/extends/GetAiCharacters.ts create mode 100644 src/onebot/action/group/GetAiRecord.ts create mode 100644 src/onebot/action/group/SendGroupAiRecord.ts diff --git a/src/core/apis/packet.ts b/src/core/apis/packet.ts index 4794f7be..a0de8a5f 100644 --- a/src/core/apis/packet.ts +++ b/src/core/apis/packet.ts @@ -1,10 +1,11 @@ +import * as crypto from 'crypto'; import * as os from 'os'; import { ChatType, InstanceContext, NapCatCore } from '..'; import offset from '@/core/external/offset.json'; import { PacketClient, RecvPacketData } from '@/core/packet/client'; import { PacketSession } from "@/core/packet/session"; import { OidbPacket, PacketHexStr } from "@/core/packet/packer"; -import { NapProtoMsg } from '@/core/packet/proto/NapProto'; +import { NapProtoEncodeStructType, NapProtoMsg } from '@/core/packet/proto/NapProto'; import { OidbSvcTrpcTcp0X9067_202_Rsp_Body } from '@/core/packet/proto/oidb/Oidb.0x9067_202'; import { OidbSvcTrpcTcpBase, OidbSvcTrpcTcpBaseRsp } from '@/core/packet/proto/oidb/OidbBase'; import { OidbSvcTrpcTcp0XFE1_2RSP } from '@/core/packet/proto/oidb/Oidb.0XFE1_2'; @@ -20,6 +21,10 @@ import { } from "@/core/packet/message/element"; import { MiniAppReqParams, MiniAppRawData } from "@/core/packet/entities/miniApp"; import { MiniAppAdaptShareInfoResp } from "@/core/packet/proto/action/miniAppAdaptShareInfo"; +import { AIVoiceChatType, AIVoiceItemList } from "@/core/packet/entities/aiChat"; +import { OidbSvcTrpcTcp0X929B_0Resp, OidbSvcTrpcTcp0X929D_0Resp } from "@/core/packet/proto/oidb/Oidb.0x929"; +import { IndexNode, MsgInfo } from "@/core/packet/proto/oidb/common/Ntv2.RichMediaReq"; +import { NTV2RichMediaResp } from "@/core/packet/proto/oidb/common/Ntv2.RichMediaResp"; interface OffsetType { @@ -188,10 +193,54 @@ export class NTQQPacketApi { return `https://${resp.download.downloadDns}/ftn_handler/${Buffer.from(resp.download.downloadUrl).toString('hex')}/?fname=`; } + async sendGroupPttFileDownloadReq(groupUin: number, node: NapProtoEncodeStructType) { + const data = this.packetSession?.packer.packGroupPttFileDownloadReq(groupUin, node); + const ret = await this.sendOidbPacket(data!, true); + const body = new NapProtoMsg(OidbSvcTrpcTcpBaseRsp).decode(Buffer.from(ret.hex_data, 'hex')).body; + const resp = new NapProtoMsg(NTV2RichMediaResp).decode(body); + const info = resp.download.info; + return `https://${info.domain}${info.urlPath}${resp.download.rKeyParam}`; + } + async sendMiniAppShareInfoReq(param: MiniAppReqParams) { const data = this.packetSession?.packer.packMiniAppAdaptShareInfo(param); const ret = await this.sendPacket("LightAppSvc.mini_app_share.AdaptShareInfo", data!, true); const body = new NapProtoMsg(MiniAppAdaptShareInfoResp).decode(Buffer.from(ret.hex_data, 'hex')); return JSON.parse(body.content.jsonContent) as MiniAppRawData; } + + async sendFetchAiVoiceListReq(groupUin: number, chatType: AIVoiceChatType) : Promise { + const data = this.packetSession?.packer.packFetchAiVoiceListReq(groupUin, chatType); + const ret = await this.sendOidbPacket(data!, true); + const body = new NapProtoMsg(OidbSvcTrpcTcpBaseRsp).decode(Buffer.from(ret.hex_data, 'hex')).body; + const resp = new NapProtoMsg(OidbSvcTrpcTcp0X929D_0Resp).decode(body); + if (!resp.content) return null; + return resp.content.map((item) => { + return { + category: item.category, + voices: item.voices + }; + }); + } + + async sendAiVoiceChatReq(groupUin: number, voiceId: string, text: string, chatType: AIVoiceChatType): Promise> { + let reqTime = 0; + const reqMaxTime = 30; + const sessionId = crypto.randomBytes(4).readUInt32BE(0); + while (true) { + if (reqTime >= reqMaxTime) { + throw new Error(`sendAiVoiceChatReq failed after ${reqMaxTime} times`); + } + reqTime++; + const data = this.packetSession?.packer.packAiVoiceChatReq(groupUin, voiceId, text, chatType, sessionId); + const ret = await this.sendOidbPacket(data!, true); + const body = new NapProtoMsg(OidbSvcTrpcTcpBase).decode(Buffer.from(ret.hex_data, 'hex')); + if (body.errorCode) { + throw new Error(`sendAiVoiceChatReq retCode: ${body.errorCode} error: ${body.errorMsg}`); + } + const resp = new NapProtoMsg(OidbSvcTrpcTcp0X929B_0Resp).decode(body.body); + if (!resp.msgInfo) continue; + return resp.msgInfo; + } + } } diff --git a/src/core/packet/entities/aiChat.ts b/src/core/packet/entities/aiChat.ts new file mode 100644 index 00000000..08f1dfda --- /dev/null +++ b/src/core/packet/entities/aiChat.ts @@ -0,0 +1,16 @@ +export enum AIVoiceChatType { + Unknown = 0, + Sound = 1, + Sing = 2 +} + +export interface AIVoiceItem { + voiceId: string; + voiceDisplayName: string; + voiceExampleUrl: string; +} + +export interface AIVoiceItemList { + category: string; + voices: AIVoiceItem[]; +} diff --git a/src/core/packet/packer.ts b/src/core/packet/packer.ts index 285eb03b..a04ec753 100644 --- a/src/core/packet/packer.ts +++ b/src/core/packet/packer.ts @@ -1,13 +1,13 @@ import * as zlib from "node:zlib"; import * as crypto from "node:crypto"; import { computeMd5AndLengthWithLimit } from "@/core/packet/utils/crypto/hash"; -import { NapProtoMsg } from "@/core/packet/proto/NapProto"; +import { NapProtoEncodeStructType, NapProtoMsg } from "@/core/packet/proto/NapProto"; import { OidbSvcTrpcTcpBase } from "@/core/packet/proto/oidb/OidbBase"; import { OidbSvcTrpcTcp0X9067_202 } from "@/core/packet/proto/oidb/Oidb.0x9067_202"; import { OidbSvcTrpcTcp0X8FC_2, OidbSvcTrpcTcp0X8FC_2_Body } from "@/core/packet/proto/oidb/Oidb.0x8FC_2"; import { OidbSvcTrpcTcp0XFE1_2 } from "@/core/packet/proto/oidb/Oidb.0XFE1_2"; import { OidbSvcTrpcTcp0XED3_1 } from "@/core/packet/proto/oidb/Oidb.0xED3_1"; -import { NTV2RichMediaReq } from "@/core/packet/proto/oidb/common/Ntv2.RichMediaReq"; +import { IndexNode, NTV2RichMediaReq } from "@/core/packet/proto/oidb/common/Ntv2.RichMediaReq"; import { HttpConn0x6ff_501 } from "@/core/packet/proto/action/action"; import { LongMsgResult, SendLongMsgReq } from "@/core/packet/proto/message/action"; import { PacketMsgBuilder } from "@/core/packet/message/builder"; @@ -28,6 +28,8 @@ import { OidbSvcTrpcTcp0XE37_800 } from "@/core/packet/proto/oidb/Oidb.0XE37_800 import { OidbSvcTrpcTcp0XEB7 } from "./proto/oidb/Oidb.0xEB7"; import { MiniAppReqParams } from "@/core/packet/entities/miniApp"; import { MiniAppAdaptShareInfoReq } from "@/core/packet/proto/action/miniAppAdaptShareInfo"; +import {AIVoiceChatType} from "@/core/packet/entities/aiChat"; +import {OidbSvcTrpcTcp0X929B_0, OidbSvcTrpcTcp0X929D_0} from "@/core/packet/proto/oidb/Oidb.0x929"; export type PacketHexStr = string & { readonly hexNya: unique symbol }; @@ -696,6 +698,37 @@ export class PacketPacker { ); } + packGroupPttFileDownloadReq(groupUin: number, node: NapProtoEncodeStructType): OidbPacket { + return this.packOidbPacket(0x126E, 200, new NapProtoMsg(NTV2RichMediaReq).encode({ + reqHead: { + common: { + requestId: 4, + command: 200 + }, + scene: { + requestType: 1, + businessType: 3, + sceneType: 2, + group: { + groupUin: groupUin + } + }, + client: { + agentType: 2 + } + }, + download: { + node: node, + download: { + video: { + busiType: 0, + sceneType: 0, + } + } + } + }), true, false); + } + packGroupSignReq(uin: string, groupCode: string): OidbPacket { return this.packOidbPacket(0XEB7, 1, new NapProtoMsg(OidbSvcTrpcTcp0XEB7).encode( { @@ -744,4 +777,27 @@ export class PacketPacker { ) ); } + + packFetchAiVoiceListReq(groupUin: number, chatType: AIVoiceChatType): OidbPacket { + return this.packOidbPacket(0x929D, 0, + new NapProtoMsg(OidbSvcTrpcTcp0X929D_0).encode({ + groupUin: groupUin, + chatType: chatType + }) + ); + } + + packAiVoiceChatReq(groupUin: number, voiceId: string, text: string, chatType: AIVoiceChatType, sessionId: number): OidbPacket { + return this.packOidbPacket(0x929B, 0, + new NapProtoMsg(OidbSvcTrpcTcp0X929B_0).encode({ + groupUin: groupUin, + voiceId: voiceId, + text: text, + chatType: chatType, + session: { + sessionId: sessionId + } + }) + ); + } } diff --git a/src/core/packet/proto/oidb/Oidb.0x929.ts b/src/core/packet/proto/oidb/Oidb.0x929.ts new file mode 100644 index 00000000..cc04e64b --- /dev/null +++ b/src/core/packet/proto/oidb/Oidb.0x929.ts @@ -0,0 +1,42 @@ +import { ScalarType } from "@protobuf-ts/runtime"; +import { ProtoField } from "../NapProto"; +import { MsgInfo } from "@/core/packet/proto/oidb/common/Ntv2.RichMediaReq"; + +export const OidbSvcTrpcTcp0X929D_0 = { + groupUin: ProtoField(1, ScalarType.UINT32), + chatType: ProtoField(2, ScalarType.UINT32), +}; + +export const OidbSvcTrpcTcp0X929D_0Resp = { + content: ProtoField(1, () => OidbSvcTrpcTcp0X929D_0RespContent, false, true), +}; + +export const OidbSvcTrpcTcp0X929D_0RespContent = { + category: ProtoField(1, ScalarType.STRING), + voices: ProtoField(2, () => OidbSvcTrpcTcp0X929D_0RespContentVoice, false, true), +}; + +export const OidbSvcTrpcTcp0X929D_0RespContentVoice = { + voiceId: ProtoField(1, ScalarType.STRING), + voiceDisplayName: ProtoField(2, ScalarType.STRING), + voiceExampleUrl: ProtoField(3, ScalarType.STRING), +}; + +export const OidbSvcTrpcTcp0X929B_0 = { + groupUin: ProtoField(1, ScalarType.UINT32), + voiceId: ProtoField(2, ScalarType.STRING), + text: ProtoField(3, ScalarType.STRING), + chatType: ProtoField(4, ScalarType.UINT32), + session: ProtoField(5, () => OidbSvcTrpcTcp0X929B_0_Session), +}; + +export const OidbSvcTrpcTcp0X929B_0_Session = { + sessionId: ProtoField(1, ScalarType.UINT32), +}; + +export const OidbSvcTrpcTcp0X929B_0Resp = { + statusCode: ProtoField(1, ScalarType.UINT32), + field2: ProtoField(2, ScalarType.UINT32, true), + field3: ProtoField(3, ScalarType.UINT32), + msgInfo: ProtoField(4, () => MsgInfo, true), +}; diff --git a/src/core/packet/proto/oidb/OidbBase.ts b/src/core/packet/proto/oidb/OidbBase.ts index 86b62b07..49e9e5cd 100644 --- a/src/core/packet/proto/oidb/OidbBase.ts +++ b/src/core/packet/proto/oidb/OidbBase.ts @@ -4,10 +4,11 @@ import { ProtoField } from "../NapProto"; export const OidbSvcTrpcTcpBase = { command: ProtoField(1, ScalarType.UINT32), subCommand: ProtoField(2, ScalarType.UINT32), + errorCode: ProtoField(3, ScalarType.UINT32), body: ProtoField(4, ScalarType.BYTES), errorMsg: ProtoField(5, ScalarType.STRING, true), isReserved: ProtoField(12, ScalarType.UINT32) }; export const OidbSvcTrpcTcpBaseRsp = { body: ProtoField(4, ScalarType.BYTES) -}; \ No newline at end of file +}; diff --git a/src/onebot/action/extends/GetAiCharacters.ts b/src/onebot/action/extends/GetAiCharacters.ts new file mode 100644 index 00000000..3a72a0b2 --- /dev/null +++ b/src/onebot/action/extends/GetAiCharacters.ts @@ -0,0 +1,41 @@ +import {ActionName} from '../types'; +import {FromSchema, JSONSchema} from 'json-schema-to-ts'; +import {GetPacketStatusDepends} from "@/onebot/action/packet/GetPacketStatus"; +import {AIVoiceChatType} from "@/core/packet/entities/aiChat"; + +const SchemaData = { + type: 'object', + properties: { + group_id: { type: ['number', 'string'] }, + chat_type: { type: ['number', 'string'] }, + }, + required: ['group_id'], +} as const satisfies JSONSchema; + +type Payload = FromSchema; + +interface GetAiCharactersResponse { + type: string; + characters: { + character_id: string; + character_name: string; + preview_url: string; + }[]; +} + +export class GetAiCharacters extends GetPacketStatusDepends { + actionName = ActionName.GetAiCharacters; + payloadSchema = SchemaData; + + async _handle(payload: Payload) { + const rawList = await this.core.apis.PacketApi.sendFetchAiVoiceListReq(+payload.group_id, +(payload.chat_type ?? 1) as AIVoiceChatType); + return rawList?.map((item) => ({ + type: item.category, + characters: item.voices.map((voice) => ({ + character_id: voice.voiceId, + character_name: voice.voiceDisplayName, + preview_url: voice.voiceExampleUrl, + })), + })) ?? []; + } +} diff --git a/src/onebot/action/group/GetAiRecord.ts b/src/onebot/action/group/GetAiRecord.ts new file mode 100644 index 00000000..5f60fc8b --- /dev/null +++ b/src/onebot/action/group/GetAiRecord.ts @@ -0,0 +1,28 @@ +import {ActionName} from '../types'; +import {FromSchema, JSONSchema} from 'json-schema-to-ts'; +import {GetPacketStatusDepends} from "@/onebot/action/packet/GetPacketStatus"; +import {AIVoiceChatType} from "@/core/packet/entities/aiChat"; +import {NapProtoEncodeStructType} from "@/core/packet/proto/NapProto"; +import {IndexNode} from "@/core/packet/proto/oidb/common/Ntv2.RichMediaReq"; + +const SchemaData = { + type: 'object', + properties: { + character: { type: ['string'] }, + group_id: { type: ['number', 'string'] }, + text: { type: 'string' }, + }, + required: ['character', 'group_id', 'text'], +} as const satisfies JSONSchema; + +type Payload = FromSchema; + +export class GetAiRecord extends GetPacketStatusDepends { + actionName = ActionName.GetAiRecord; + payloadSchema = SchemaData; + + async _handle(payload: Payload) { + const rawRsp = await this.core.apis.PacketApi.sendAiVoiceChatReq(+payload.group_id, payload.character, payload.text, AIVoiceChatType.Sound); + return await this.core.apis.PacketApi.sendGroupPttFileDownloadReq(+payload.group_id, rawRsp.msgInfoBody![0].index as NapProtoEncodeStructType); + } +} diff --git a/src/onebot/action/group/SendGroupAiRecord.ts b/src/onebot/action/group/SendGroupAiRecord.ts new file mode 100644 index 00000000..173eee80 --- /dev/null +++ b/src/onebot/action/group/SendGroupAiRecord.ts @@ -0,0 +1,40 @@ +import {ActionName} from '../types'; +import {FromSchema, JSONSchema} from 'json-schema-to-ts'; +import {GetPacketStatusDepends} from "@/onebot/action/packet/GetPacketStatus"; +import {AIVoiceChatType} from "@/core/packet/entities/aiChat"; +import {uri2local} from "@/common/file"; +import {ChatType, Peer} from "@/core"; +import {NapProtoEncodeStructType} from "@/core/packet/proto/NapProto"; +import {IndexNode} from "@/core/packet/proto/oidb/common/Ntv2.RichMediaReq"; + +const SchemaData = { + type: 'object', + properties: { + character: { type: ['string'] }, + group_id: { type: ['number', 'string'] }, + text: { type: 'string' }, + }, + required: ['character', 'group_id', 'text'], +} as const satisfies JSONSchema; + +type Payload = FromSchema; + +export class SendGroupAiRecord extends GetPacketStatusDepends { + actionName = ActionName.SendGroupAiRecord; + payloadSchema = SchemaData; + + async _handle(payload: Payload) { + const rawRsp = await this.core.apis.PacketApi.sendAiVoiceChatReq(+payload.group_id, payload.character, payload.text, AIVoiceChatType.Sound); + const url = await this.core.apis.PacketApi.sendGroupPttFileDownloadReq(+payload.group_id, rawRsp.msgInfoBody![0].index as NapProtoEncodeStructType); + const { path, fileName, errMsg, success} = (await uri2local(this.core.NapCatTempPath, url)); + if (!success) { + throw new Error(errMsg); + } + const peer = {chatType: ChatType.KCHATTYPEGROUP, peerUid: payload.group_id.toString()} as Peer; + const element = await this.core.apis.FileApi.createValidSendPttElement(path); + const sendRes = await this.obContext.apis.MsgApi.sendMsgWithOb11UniqueId(peer, [element], [path]); + return {message_id: sendRes.msgId}; + } +} diff --git a/src/onebot/action/index.ts b/src/onebot/action/index.ts index 12ea7127..72b96d9e 100644 --- a/src/onebot/action/index.ts +++ b/src/onebot/action/index.ts @@ -99,6 +99,9 @@ import { GoCQHTTPGetModelShow } from './go-cqhttp/GoCQHTTPGetModelShow'; import { GoCQHTTPSetModelShow } from './go-cqhttp/GoCQHTTPSetModelShow'; import { GoCQHTTPDeleteFriend } from './go-cqhttp/GoCQHTTPDeleteFriend'; import { GetMiniAppArk } from "@/onebot/action/extends/GetMiniAppArk"; +import { GetAiRecord } from "@/onebot/action/group/GetAiRecord"; +import { SendGroupAiRecord } from "@/onebot/action/group/SendGroupAiRecord"; +import { GetAiCharacters } from "@/onebot/action/extends/GetAiCharacters"; export type ActionMap = Map>; @@ -212,6 +215,9 @@ export function createActionMap(obContext: NapCatOneBot11Adapter, core: NapCatCo new GetGroupShutList(obContext, core), new GetGroupFileUrl(obContext, core), new GetMiniAppArk(obContext, core), + new GetAiRecord(obContext, core), + new SendGroupAiRecord(obContext, core), + new GetAiCharacters(obContext, core), ]; const actionMap = new Map(); for (const action of actionHandlers) { diff --git a/src/onebot/action/types.ts b/src/onebot/action/types.ts index 5c998a47..effd446a 100644 --- a/src/onebot/action/types.ts +++ b/src/onebot/action/types.ts @@ -138,4 +138,7 @@ export enum ActionName { SetGroupSign = "set_group_sign", GetMiniAppArk = "get_mini_app_ark", // UploadForwardMsg = "upload_forward_msg", + GetAiRecord = "get_ai_record", + GetAiCharacters = "get_ai_characters", + SendGroupAiRecord = "send_group_ai_record", }