From c8ee3719820ed7452127ff9ada8029dc430a889c Mon Sep 17 00:00:00 2001 From: pk5ls20 Date: Sun, 12 Jan 2025 10:13:12 +0800 Subject: [PATCH] feat: packet ocr --- src/core/packet/context/operationContext.ts | 42 +++++++++++++ src/core/packet/entities/ocrResult.ts | 15 +++++ .../packet/transformer/action/ImageOCR.ts | 37 ++++++++++++ src/core/packet/transformer/action/index.ts | 1 + .../transformer/highway/DownloadImage.ts | 51 ++++++++++++++++ src/core/packet/transformer/highway/index.ts | 1 + src/core/packet/transformer/proto/index.ts | 1 + .../transformer/proto/oidb/Oidb.0xE07.ts | 59 +++++++++++++++++++ 8 files changed, 207 insertions(+) create mode 100644 src/core/packet/entities/ocrResult.ts create mode 100644 src/core/packet/transformer/action/ImageOCR.ts create mode 100644 src/core/packet/transformer/highway/DownloadImage.ts create mode 100644 src/core/packet/transformer/proto/oidb/Oidb.0xE07.ts diff --git a/src/core/packet/context/operationContext.ts b/src/core/packet/context/operationContext.ts index 2626ac2f..0fe50b62 100644 --- a/src/core/packet/context/operationContext.ts +++ b/src/core/packet/context/operationContext.ts @@ -14,9 +14,11 @@ import { AIVoiceChatType } from "@/core/packet/entities/aiChat"; import { NapProtoDecodeStructType, NapProtoEncodeStructType } from "@napneko/nap-proto-core"; import { IndexNode, MsgInfo } from "@/core/packet/transformer/proto"; import { OidbPacket } from "@/core/packet/transformer/base"; +import { ImageOcrResult } from "@/core/packet/entities/ocrResult"; export class PacketOperationContext { private readonly context: PacketContext; + constructor(context: PacketContext) { this.context = context; } @@ -95,6 +97,46 @@ export class PacketOperationContext { }); } + async UploadImage(img: PacketMsgPicElement) { + await this.context.highway.uploadImage({ + chatType: ChatType.KCHATTYPEC2C, + peerUid: this.context.napcore.basicInfo.uid + }, img); + const index = img.msgInfo?.msgInfoBody?.at(0)?.index; + if (!index) { + throw new Error('img.msgInfo?.msgInfoBody![0].index! is undefined'); + } + return await this.GetImageUrl(this.context.napcore.basicInfo.uid, index); + } + + async GetImageUrl(selfUid: string, node: NapProtoEncodeStructType) { + const req = trans.DownloadImage.build(selfUid, node); + const resp = await this.context.client.sendOidbPacket(req, true); + const res = trans.DownloadImage.parse(resp); + return `https://${res.download.info.domain}${res.download.info.urlPath}${res.download.rKeyParam}`; + } + + async ImageOCR(imgUrl: string) { + const req = trans.ImageOCR.build(imgUrl); + const resp = await this.context.client.sendOidbPacket(req, true); + const res = trans.ImageOCR.parse(resp); + return { + texts: res.ocrRspBody.textDetections.map((item) => { + return { + text: item.detectedText, + confidence: item.confidence, + coordinates: item.polygon.coordinates.map((c) => { + return { + x: c.x, + y: c.y + }; + }), + }; + }), + language: res.ocrRspBody.language + } as ImageOcrResult; + } + async UploadForwardMsg(msg: PacketMsg[], groupUin: number = 0) { await this.UploadResources(msg, groupUin); const req = trans.UploadForwardMsg.build(this.context.napcore.basicInfo.uid, msg, groupUin); diff --git a/src/core/packet/entities/ocrResult.ts b/src/core/packet/entities/ocrResult.ts new file mode 100644 index 00000000..31fdefa8 --- /dev/null +++ b/src/core/packet/entities/ocrResult.ts @@ -0,0 +1,15 @@ +export interface ImageOcrResult { + texts: TextDetection[]; + language: string; +} + +export interface TextDetection { + text: string; + confidence: number; + coordinates: Coordinate[]; +} + +export interface Coordinate { + x: number; + y: number; +} diff --git a/src/core/packet/transformer/action/ImageOCR.ts b/src/core/packet/transformer/action/ImageOCR.ts new file mode 100644 index 00000000..1f05a91c --- /dev/null +++ b/src/core/packet/transformer/action/ImageOCR.ts @@ -0,0 +1,37 @@ +import * as proto from "@/core/packet/transformer/proto"; +import { NapProtoMsg } from "@napneko/nap-proto-core"; +import { OidbPacket, PacketTransformer } from "@/core/packet/transformer/base"; +import OidbBase from "@/core/packet/transformer/oidb/oidbBase"; + +class ImageOCR extends PacketTransformer { + constructor() { + super(); + } + + build(url: string): OidbPacket { + const body = new NapProtoMsg(proto.OidbSvcTrpcTcp0xE07_0).encode( + { + version: 1, + client: 0, + entrance: 1, + ocrReqBody: { + imageUrl: url, + originMd5: "", + afterCompressMd5: "", + afterCompressFileSize: "", + afterCompressWeight: "", + afterCompressHeight: "", + isCut: false, + } + } + ); + return OidbBase.build(0XEB7, 1, body, false, false); + } + + parse(data: Buffer) { + const base = OidbBase.parse(data); + return new NapProtoMsg(proto.OidbSvcTrpcTcp0xE07_0_Response).decode(base.body); + } +} + +export default new ImageOCR(); diff --git a/src/core/packet/transformer/action/index.ts b/src/core/packet/transformer/action/index.ts index a92c7321..7f0987d6 100644 --- a/src/core/packet/transformer/action/index.ts +++ b/src/core/packet/transformer/action/index.ts @@ -5,3 +5,4 @@ export { default as GroupSign } from './GroupSign'; export { default as GetStrangerInfo } from './GetStrangerInfo'; export { default as SendPoke } from './SendPoke'; export { default as SetSpecialTitle } from './SetSpecialTitle'; +export { default as ImageOCR } from './ImageOCR'; diff --git a/src/core/packet/transformer/highway/DownloadImage.ts b/src/core/packet/transformer/highway/DownloadImage.ts new file mode 100644 index 00000000..8d99fa2d --- /dev/null +++ b/src/core/packet/transformer/highway/DownloadImage.ts @@ -0,0 +1,51 @@ +import * as proto from "@/core/packet/transformer/proto"; +import { NapProtoEncodeStructType, NapProtoMsg } from "@napneko/nap-proto-core"; +import { OidbPacket, PacketTransformer } from "@/core/packet/transformer/base"; +import OidbBase from "@/core/packet/transformer/oidb/oidbBase"; +import { IndexNode } from "@/core/packet/transformer/proto"; + +class DownloadImage extends PacketTransformer { + constructor() { + super(); + } + + build(selfUid: string, node: NapProtoEncodeStructType): OidbPacket { + const body = new NapProtoMsg(proto.NTV2RichMediaReq).encode({ + reqHead: { + common: { + requestId: 1, + command: 100 + }, + scene: { + requestType: 2, + businessType: 1, + sceneType: 1, + c2C: { + accountType: 2, + targetUid: selfUid + }, + }, + client: { + agentType: 2, + } + }, + download: { + node: node, + download: { + video: { + busiType: 0, + sceneType: 0 + } + } + } + }); + return OidbBase.build(0x11C5, 200, body, true, false); + } + + parse(data: Buffer) { + const oidbBody = OidbBase.parse(data).body; + return new NapProtoMsg(proto.NTV2RichMediaResp).decode(oidbBody); + } +} + +export default new DownloadImage(); diff --git a/src/core/packet/transformer/highway/index.ts b/src/core/packet/transformer/highway/index.ts index 9444789d..f9170cea 100644 --- a/src/core/packet/transformer/highway/index.ts +++ b/src/core/packet/transformer/highway/index.ts @@ -11,3 +11,4 @@ export { default as UploadPrivateFile } from './UploadPrivateFile'; export { default as UploadPrivateImage } from './UploadPrivateImage'; export { default as UploadPrivatePtt } from './UploadPrivatePtt'; export { default as UploadPrivateVideo } from './UploadPrivateVideo'; +export { default as DownloadImage } from './DownloadImage'; diff --git a/src/core/packet/transformer/proto/index.ts b/src/core/packet/transformer/proto/index.ts index da8f1293..4e2aa3a8 100644 --- a/src/core/packet/transformer/proto/index.ts +++ b/src/core/packet/transformer/proto/index.ts @@ -29,3 +29,4 @@ export * from "./oidb/Oidb.0xEB7"; export * from "./oidb/Oidb.0xED3_1"; export * from "./oidb/Oidb.0XFE1_2"; export * from "./oidb/OidbBase"; +export * from "./oidb/Oidb.0xE07"; diff --git a/src/core/packet/transformer/proto/oidb/Oidb.0xE07.ts b/src/core/packet/transformer/proto/oidb/Oidb.0xE07.ts new file mode 100644 index 00000000..f94bd18e --- /dev/null +++ b/src/core/packet/transformer/proto/oidb/Oidb.0xE07.ts @@ -0,0 +1,59 @@ +import { ProtoField, ScalarType } from "@napneko/nap-proto-core"; + +export const OidbSvcTrpcTcp0xE07_0 = { + version: ProtoField(1, ScalarType.UINT32), + client: ProtoField(2, ScalarType.UINT32), + entrance: ProtoField(3, ScalarType.UINT32), + ocrReqBody: ProtoField(10, () => OcrReqBody, true), +}; + +export const OcrReqBody = { + imageUrl: ProtoField(1, ScalarType.STRING), + languageType: ProtoField(2, ScalarType.UINT32), + scene: ProtoField(3, ScalarType.UINT32), + originMd5: ProtoField(10, ScalarType.STRING), + afterCompressMd5: ProtoField(11, ScalarType.STRING), + afterCompressFileSize: ProtoField(12, ScalarType.STRING), + afterCompressWeight: ProtoField(13, ScalarType.STRING), + afterCompressHeight: ProtoField(14, ScalarType.STRING), + isCut: ProtoField(15, ScalarType.BOOL), +}; + +export const OidbSvcTrpcTcp0xE07_0_Response = { + retCode: ProtoField(1, ScalarType.INT32), + errMsg: ProtoField(2, ScalarType.STRING), + wording: ProtoField(3, ScalarType.STRING), + ocrRspBody: ProtoField(10, () => OcrRspBody), +}; + +export const OcrRspBody = { + textDetections: ProtoField(1, () => TextDetection, false, true), + language: ProtoField(2, ScalarType.STRING), + requestId: ProtoField(3, ScalarType.STRING), + ocrLanguageList: ProtoField(101, ScalarType.STRING, false, true), + dstTranslateLanguageList: ProtoField(102, ScalarType.STRING, false, true), + languageList: ProtoField(103, () => Language, false, true), + afterCompressWeight: ProtoField(111, ScalarType.UINT32), + afterCompressHeight: ProtoField(112, ScalarType.UINT32), +}; + +export const TextDetection = { + detectedText: ProtoField(1, ScalarType.STRING), + confidence: ProtoField(2, ScalarType.UINT32), + polygon: ProtoField(3, () => Polygon), + advancedInfo: ProtoField(4, ScalarType.STRING), +}; + +export const Polygon = { + coordinates: ProtoField(1, () => Coordinate, false, true), +}; + +export const Coordinate = { + x: ProtoField(1, ScalarType.INT32), + y: ProtoField(2, ScalarType.INT32), +}; + +export const Language = { + languageCode: ProtoField(1, ScalarType.STRING), + languageDesc: ProtoField(2, ScalarType.STRING), +};