插件权限设计指南

import { Injectable } from '@nestjs/common';
import { Document } from 'langchain/document';
import {
  ImageUnderstandingStrategy,
  IImageUnderstandingStrategy,
  TImageUnderstandingFile,
  TImageUnderstandingResult
} from './strategy.interface';

@Injectable()
@ImageUnderstandingStrategy('openai-gpt4v')
export class OpenAIGPT4VStrategy implements IImageUnderstandingStrategy<any> {
  readonly meta = {
    name: 'openai-gpt4v',
    label: { en: 'OpenAI GPT-4 Vision', zh: 'OpenAI GPT-4 图像理解' },

    /**
     * 权限声明：告诉核心系统，这个插件需要一个 "llm" 权限，
     * 提供方是 openai，能力是 vision。
     * 核心系统识别后会根据 config.provider + config.model 实例化对应的 Client，
     * 并注入到 config.client
     */
    permissions: [
      {
        type: 'llm',
        provider: 'openai',
        capability: 'vision',
      },
    ],

    /**
     * 配置 schema，用户需要填写 provider 和 model，
     * 核心系统会根据 provider + model + 用户账号 API Key，
     * 实例化好 Client 注入。
     */
    configSchema: {
      type: 'object',
      properties: {
        provider: { 
          type: 'string', 
          enum: ['openai'], 
          description: '模型提供商' 
        },
        model: { 
          type: 'string', 
          default: 'gpt-4-vision-preview', 
          description: '大模型名称' 
        },
        prompt: { 
          type: 'string', 
          default: 'Describe this image in detail.', 
          description: '提示词（可以自定义理解任务）' 
        }
      },
      required: ['provider', 'model']
    },

    icon: {
      svg: '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><circle cx="10" cy="10" r="8" stroke="black" fill="none"/><text x="6" y="14">VLM</text></svg>',
      color: '#10a37f'
    }
  };

  async validateConfig(config: any): Promise<void> {
    if (!config.client) {
      throw new Error('Missing injected LLM client in config. Core system must inject client.');
    }
  }

  async understandImages(
    files: TImageUnderstandingFile[],
    config: any
  ): Promise<TImageUnderstandingResult[]> {
    const client = config.client; // ✅ 已由核心系统注入
    const results: TImageUnderstandingResult[] = [];

    for (const file of files) {
      const description = await this.runGPT4V(client, file.path, config);

      const doc = new Document({
        pageContent: description,
        metadata: {
          chunkId: `img-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
          parentChunkId: file.parentChunkId,
          imagePath: file.path,
          source: file.filename,
          type: 'vlm',
          engine: config.model
        }
      });

      results.push({
        docs: [doc],
        metadata: { engine: config.model, file: file.filename }
      });
    }

    return results;
  }

  private async runGPT4V(client: any, imagePath: string, config: any): Promise<string> {
    // client 已经是 openai client，由核心系统实例化
    const response = await client.chat.completions.create({
      model: config.model,
      messages: [
        {
          role: 'user',
          content: [
            { type: 'text', text: config.prompt || 'Describe this image in detail.' },
            { type: 'image_url', image_url: { url: imagePath } }
          ]
        }
      ],
      max_tokens: 500
    });

    return response.choices[0]?.message?.content || '';
  }
}

const strategy = this.imageUnderstandingRegistry.get('openai-gpt4v');
const result = await strategy.understandImages(
  [
    { 
      path: 'https://example.com/chart.png', 
      filename: 'chart.png', 
      extname: 'png', 
      parentChunkId: 'uuid-123' 
    }
  ],
  { 
    provider: 'openai',
    model: 'gpt-4-vision-preview',
    prompt: '请用中文详细描述这张图片。',
    client: openaiClient // ✅ 核心系统注入
  }
);

console.log(result[0].docs[0].pageContent);