400 lines
14 KiB
JavaScript
400 lines
14 KiB
JavaScript
|
#!/usr/bin/env node
|
|||
|
|
'use strict';
|
|||
|
|
|
|||
|
|
const fs = require('node:fs');
|
|||
|
|
const os = require('node:os');
|
|||
|
|
const path = require('node:path');
|
|||
|
|
|
|||
|
|
const DEFAULT_PROMPT = `你是专业汽车环车详细检测专家。以下是汽车视频中抽取的帧画面，请对每一帧进行全面的环车详细检测分析。
|
|||
|
|
|
|||
|
|
你需要同时完成以下所有检测项目：
|
|||
|
|
|
|||
|
|
## 一、车身旧伤检测
|
|||
|
|
仔细检查每一帧画面中车身的所有可见部位，识别划痕、凹陷、裂纹、掉漆、锈蚀等损伤。
|
|||
|
|
|
|||
|
|
## 二、车辆关键信息识别
|
|||
|
|
请在帧画面中识别车辆品牌型号、前后车牌号、挡风玻璃VIN、铭牌VIN、仪表盘总里程、4条胎胎压、4个轮胎纹理健康度、4个轮胎品牌型号规格。
|
|||
|
|
|
|||
|
|
要求：
|
|||
|
|
1. 仔细检查每一帧画面，不要遗漏。
|
|||
|
|
2. 不可见字段填null，不要猜测。
|
|||
|
|
3. 部分遮挡字符用"?"。
|
|||
|
|
4. 不要把反光、阴影、污渍、压缩噪声虚报为旧伤。
|
|||
|
|
5. 只输出合法JSON。
|
|||
|
|
|
|||
|
|
返回格式：
|
|||
|
|
{
|
|||
|
|
"has_damage": true,
|
|||
|
|
"damages": [
|
|||
|
|
{
|
|||
|
|
"time_second": 12.4,
|
|||
|
|
"location": "左前翼子板",
|
|||
|
|
"type": "划痕",
|
|||
|
|
"severity": "轻微",
|
|||
|
|
"description": "详细描述"
|
|||
|
|
}
|
|||
|
|
],
|
|||
|
|
"vehicle_info": {
|
|||
|
|
"vehicle_brand_model": null,
|
|||
|
|
"front_plate": null,
|
|||
|
|
"rear_plate": null,
|
|||
|
|
"windshield_vin": null,
|
|||
|
|
"nameplate_vin": null,
|
|||
|
|
"odometer_km": null,
|
|||
|
|
"tire_pressure": {"left_front": null, "right_front": null, "left_rear": null, "right_rear": null},
|
|||
|
|
"tire_tread_health": {"left_front": null, "right_front": null, "left_rear": null, "right_rear": null},
|
|||
|
|
"tire_specs": {"left_front": null, "right_front": null, "left_rear": null, "right_rear": null}
|
|||
|
|
},
|
|||
|
|
"summary": "本批检测结果简述"
|
|||
|
|
}`;
|
|||
|
|
|
|||
|
|
function emit(stage, message, status = 'running', extra = {}) {
|
|||
|
|
process.stderr.write(JSON.stringify({
|
|||
|
|
type: 'process_event',
|
|||
|
|
stage,
|
|||
|
|
message,
|
|||
|
|
status,
|
|||
|
|
timestamp: new Date().toISOString(),
|
|||
|
|
...extra,
|
|||
|
|
}) + '\n');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function readInput() {
|
|||
|
|
const raw = String(process.argv[2] || process.env.SKILL_INPUT || process.env.AIFLOW_SKILL_INPUT || fs.readFileSync(0, 'utf8')).trim();
|
|||
|
|
return raw ? JSON.parse(raw) : {};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function clampNumber(value, min, max, fallback) {
|
|||
|
|
const number = Number(value);
|
|||
|
|
if (!Number.isFinite(number)) return fallback;
|
|||
|
|
return Math.max(min, Math.min(max, number));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function workspaceRoot(env) {
|
|||
|
|
if (env.VEHICLE_SCRATCH_WORKSPACE_ROOT) return path.resolve(env.VEHICLE_SCRATCH_WORKSPACE_ROOT);
|
|||
|
|
if (env.RZYX_AI_WORKSPACE_ROOT) {
|
|||
|
|
const root = path.resolve(env.RZYX_AI_WORKSPACE_ROOT);
|
|||
|
|
return path.basename(root) === 'vehicle-scratch-inspection' ? root : path.join(root, 'vehicle-scratch-inspection');
|
|||
|
|
}
|
|||
|
|
if (env.RZYX_AI_DATA_DIR) return path.join(path.resolve(env.RZYX_AI_DATA_DIR), 'workspace', 'vehicle-scratch-inspection');
|
|||
|
|
return path.join(os.tmpdir(), 'vehicle-scratch-inspection');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function workspaceFor(taskId, env) {
|
|||
|
|
const safe = String(taskId || '').trim();
|
|||
|
|
if (!/^[a-zA-Z0-9_-]{1,80}$/.test(safe)) throw new Error('taskId非法或为空');
|
|||
|
|
const workspacePath = path.join(workspaceRoot(env), safe);
|
|||
|
|
if (!fs.existsSync(workspacePath)) throw new Error(`workspace不存在: ${workspacePath}`);
|
|||
|
|
return { taskId: safe, workspacePath };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function readJson(filePath, fallback = null) {
|
|||
|
|
try {
|
|||
|
|
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|||
|
|
} catch {
|
|||
|
|
return fallback;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function writeJson(filePath, value) {
|
|||
|
|
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|||
|
|
fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function parseJsonLoose(value) {
|
|||
|
|
if (!value) return null;
|
|||
|
|
if (typeof value === 'object') return value;
|
|||
|
|
const text = String(value).trim();
|
|||
|
|
const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i);
|
|||
|
|
const body = fenced ? fenced[1].trim() : text;
|
|||
|
|
try {
|
|||
|
|
return JSON.parse(body);
|
|||
|
|
} catch {
|
|||
|
|
const start = body.indexOf('{');
|
|||
|
|
const end = body.lastIndexOf('}');
|
|||
|
|
if (start >= 0 && end > start) {
|
|||
|
|
try { return JSON.parse(body.slice(start, end + 1)); } catch { return null; }
|
|||
|
|
}
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function chunkArray(items, size) {
|
|||
|
|
const out = [];
|
|||
|
|
for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size));
|
|||
|
|
return out;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function sleep(ms) {
|
|||
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function runWithConcurrency(tasks, limit) {
|
|||
|
|
const results = new Array(tasks.length);
|
|||
|
|
let next = 0;
|
|||
|
|
async function worker() {
|
|||
|
|
while (next < tasks.length) {
|
|||
|
|
const index = next++;
|
|||
|
|
results[index] = await tasks[index]();
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
await Promise.all(Array.from({ length: Math.min(limit, tasks.length) }, worker));
|
|||
|
|
return results;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function emptyVehicleInfo() {
|
|||
|
|
return {
|
|||
|
|
vehicleBrandModel: null,
|
|||
|
|
frontPlate: null,
|
|||
|
|
rearPlate: null,
|
|||
|
|
windshieldVin: null,
|
|||
|
|
nameplateVin: null,
|
|||
|
|
odometerKm: null,
|
|||
|
|
tirePressure: { leftFront: null, rightFront: null, leftRear: null, rightRear: null },
|
|||
|
|
tireTreadHealth: { leftFront: null, rightFront: null, leftRear: null, rightRear: null },
|
|||
|
|
tireSpecs: { leftFront: null, rightFront: null, leftRear: null, rightRear: null },
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const VEHICLE_KEY_MAP = {
|
|||
|
|
vehicle_brand_model: 'vehicleBrandModel',
|
|||
|
|
vehicleBrandModel: 'vehicleBrandModel',
|
|||
|
|
front_plate: 'frontPlate',
|
|||
|
|
frontPlate: 'frontPlate',
|
|||
|
|
rear_plate: 'rearPlate',
|
|||
|
|
rearPlate: 'rearPlate',
|
|||
|
|
windshield_vin: 'windshieldVin',
|
|||
|
|
windshieldVin: 'windshieldVin',
|
|||
|
|
nameplate_vin: 'nameplateVin',
|
|||
|
|
nameplateVin: 'nameplateVin',
|
|||
|
|
odometer_km: 'odometerKm',
|
|||
|
|
odometerKm: 'odometerKm',
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const TIRE_GROUP_MAP = {
|
|||
|
|
tire_pressure: 'tirePressure',
|
|||
|
|
tirePressure: 'tirePressure',
|
|||
|
|
tire_tread_health: 'tireTreadHealth',
|
|||
|
|
tireTreadHealth: 'tireTreadHealth',
|
|||
|
|
tire_specs: 'tireSpecs',
|
|||
|
|
tireSpecs: 'tireSpecs',
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const TIRE_POS_MAP = {
|
|||
|
|
left_front: 'leftFront',
|
|||
|
|
leftFront: 'leftFront',
|
|||
|
|
LF: 'leftFront',
|
|||
|
|
right_front: 'rightFront',
|
|||
|
|
rightFront: 'rightFront',
|
|||
|
|
RF: 'rightFront',
|
|||
|
|
left_rear: 'leftRear',
|
|||
|
|
leftRear: 'leftRear',
|
|||
|
|
LR: 'leftRear',
|
|||
|
|
right_rear: 'rightRear',
|
|||
|
|
rightRear: 'rightRear',
|
|||
|
|
RR: 'rightRear',
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
function mergeVehicleInfo(target, raw) {
|
|||
|
|
if (!raw || typeof raw !== 'object') return target;
|
|||
|
|
for (const [sourceKey, targetKey] of Object.entries(VEHICLE_KEY_MAP)) {
|
|||
|
|
const value = raw[sourceKey];
|
|||
|
|
if ((target[targetKey] === null || target[targetKey] === '' || target[targetKey] === undefined) && value !== null && value !== undefined && value !== '') {
|
|||
|
|
target[targetKey] = value;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
for (const [sourceGroup, targetGroup] of Object.entries(TIRE_GROUP_MAP)) {
|
|||
|
|
const group = raw[sourceGroup];
|
|||
|
|
if (!group || typeof group !== 'object') continue;
|
|||
|
|
for (const [sourcePos, targetPos] of Object.entries(TIRE_POS_MAP)) {
|
|||
|
|
const value = group[sourcePos];
|
|||
|
|
if (!target[targetGroup][targetPos] && value) target[targetGroup][targetPos] = value;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return target;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function normalizeDamage(raw, batchNo, index) {
|
|||
|
|
const ts = Number(raw.time_second ?? raw.timeSecond ?? raw.timestamp ?? raw.time ?? 0);
|
|||
|
|
return {
|
|||
|
|
id: raw.id ? String(raw.id) : `damage_raw_${String(batchNo).padStart(2, '0')}_${String(index + 1).padStart(3, '0')}`,
|
|||
|
|
timeSecond: Number.isFinite(ts) ? Number(ts.toFixed(3)) : 0,
|
|||
|
|
location: String(raw.location || raw.part || raw.position || '未知部位').trim() || '未知部位',
|
|||
|
|
type: String(raw.type || raw.damageType || '旧伤').trim() || '旧伤',
|
|||
|
|
severity: String(raw.severity || '轻微').trim() || '轻微',
|
|||
|
|
description: String(raw.description || raw.detail || '').trim(),
|
|||
|
|
sourceBatch: batchNo,
|
|||
|
|
raw,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function normalizeBatchResult(raw, batchNo) {
|
|||
|
|
const parsed = parseJsonLoose(raw) || {};
|
|||
|
|
const damages = Array.isArray(parsed.damages) ? parsed.damages.map((item, index) => normalizeDamage(item, batchNo, index)) : [];
|
|||
|
|
return {
|
|||
|
|
batch: batchNo,
|
|||
|
|
success: true,
|
|||
|
|
hasDamage: Boolean(parsed.has_damage ?? parsed.hasDamage ?? damages.length > 0),
|
|||
|
|
damages,
|
|||
|
|
vehicleInfo: parsed.vehicle_info || parsed.vehicleInfo || {},
|
|||
|
|
summary: parsed.summary || '',
|
|||
|
|
raw: parsed,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function collectModelResults(input) {
|
|||
|
|
if (Array.isArray(input.batchResults)) return input.batchResults;
|
|||
|
|
const candidate = input.modelResult ?? input.detectResult ?? input.result ?? input.mockResult;
|
|||
|
|
if (!candidate) return null;
|
|||
|
|
const parsed = parseJsonLoose(candidate);
|
|||
|
|
if (Array.isArray(parsed)) return parsed;
|
|||
|
|
if (Array.isArray(parsed?.batches)) return parsed.batches;
|
|||
|
|
if (Array.isArray(parsed?.batchResults)) return parsed.batchResults;
|
|||
|
|
return [parsed || candidate];
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function imageContent(filePath) {
|
|||
|
|
return {
|
|||
|
|
type: 'image_url',
|
|||
|
|
image_url: { url: `data:image/jpeg;base64,${fs.readFileSync(filePath).toString('base64')}` },
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function callVisionApi({ env, content }) {
|
|||
|
|
const apiKey = env.ARK_API_KEY;
|
|||
|
|
if (!apiKey) throw new Error('缺少ARK_API_KEY');
|
|||
|
|
const apiUrl = env.ARK_API_URL || 'https://ark.cn-beijing.volces.com/api/v3/chat/completions';
|
|||
|
|
const model = env.DAMAGE_DETECT_MODEL || 'doubao-seed-2-0-pro-260215';
|
|||
|
|
for (let attempt = 1; attempt <= 4; attempt += 1) {
|
|||
|
|
emit('vision-api', '请求多模态检测模型', 'running', { model, attempt, totalAttempts: 4 });
|
|||
|
|
try {
|
|||
|
|
const response = await fetch(apiUrl, {
|
|||
|
|
method: 'POST',
|
|||
|
|
headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' },
|
|||
|
|
body: JSON.stringify({
|
|||
|
|
model,
|
|||
|
|
messages: [{ role: 'user', content }],
|
|||
|
|
max_tokens: 4096,
|
|||
|
|
temperature: 0.1,
|
|||
|
|
}),
|
|||
|
|
signal: AbortSignal.timeout(600000),
|
|||
|
|
});
|
|||
|
|
const text = await response.text();
|
|||
|
|
let body;
|
|||
|
|
try { body = JSON.parse(text); } catch { body = { raw: text }; }
|
|||
|
|
if (!response.ok) throw new Error(JSON.stringify(body).slice(0, 1000));
|
|||
|
|
return body.choices?.[0]?.message?.content || '';
|
|||
|
|
} catch (err) {
|
|||
|
|
const msg = err && err.message ? err.message : String(err);
|
|||
|
|
if (attempt < 4 && /429|TooManyRequests|timeout|ECONNRESET|ETIMEDOUT/i.test(msg)) {
|
|||
|
|
await sleep(3000 * attempt);
|
|||
|
|
continue;
|
|||
|
|
}
|
|||
|
|
throw err;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
throw new Error('多模态检测模型调用失败');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function detectBatch({ batch, batchNo, totalBatches, env, prompt }) {
|
|||
|
|
emit('detect_damages', '开始检测批次', 'running', { batch: batchNo, totalBatches, frameCount: batch.length });
|
|||
|
|
const content = [];
|
|||
|
|
for (const frame of batch) {
|
|||
|
|
content.push({ type: 'text', text: `[${Number(frame.timestamp).toFixed(2)} second] frame ${frame.index}` });
|
|||
|
|
content.push(imageContent(frame.path));
|
|||
|
|
}
|
|||
|
|
content.push({ type: 'text', text: prompt });
|
|||
|
|
const modelText = await callVisionApi({ env, content });
|
|||
|
|
const result = normalizeBatchResult(modelText, batchNo);
|
|||
|
|
emit('detect_damages', '批次检测完成', 'running', { batch: batchNo, totalBatches, damageCount: result.damages.length });
|
|||
|
|
return result;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function buildVisionRequestOutput({ workspace, batches, prompt }) {
|
|||
|
|
return {
|
|||
|
|
success: false,
|
|||
|
|
needsModelVision: true,
|
|||
|
|
taskId: workspace.taskId,
|
|||
|
|
workspacePath: workspace.workspacePath,
|
|||
|
|
prompt,
|
|||
|
|
batches: batches.map((batch, index) => ({
|
|||
|
|
batch: index + 1,
|
|||
|
|
frames: batch.map(frame => ({
|
|||
|
|
index: frame.index,
|
|||
|
|
timestamp: frame.timestamp,
|
|||
|
|
path: frame.path,
|
|||
|
|
relativePath: frame.relativePath,
|
|||
|
|
})),
|
|||
|
|
})),
|
|||
|
|
error: '未配置ARK_API_KEY且未提供modelResult/batchResults。请让多模态模型按prompt分析每批图片，并把JSON结果作为batchResults传回本skill。',
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function run(input, env = process.env) {
|
|||
|
|
if (!input?.taskId) throw new Error('taskId不能为空，请先使用video-frame-extractor抽帧');
|
|||
|
|
const workspace = workspaceFor(input.taskId, env);
|
|||
|
|
const videoInfoDoc = readJson(path.join(workspace.workspacePath, 'video_info.json'));
|
|||
|
|
if (!videoInfoDoc?.frames?.length) throw new Error('未找到video_info.json或frames为空，请先抽帧');
|
|||
|
|
const batchSize = Math.round(clampNumber(input.batchSize, 1, 100, 50));
|
|||
|
|
const batchDelay = clampNumber(input.batchDelay, 0, 30, 2);
|
|||
|
|
const concurrency = Math.round(clampNumber(input.concurrency, 1, 8, 5));
|
|||
|
|
const frames = videoInfoDoc.frames.map(frame => ({
|
|||
|
|
...frame,
|
|||
|
|
path: path.isAbsolute(frame.path) ? frame.path : path.join(workspace.workspacePath, frame.relativePath || frame.path),
|
|||
|
|
}));
|
|||
|
|
const batches = chunkArray(frames, batchSize);
|
|||
|
|
const prompt = input.prompt ? String(input.prompt) : DEFAULT_PROMPT;
|
|||
|
|
|
|||
|
|
let batchResults;
|
|||
|
|
const externalResults = collectModelResults(input);
|
|||
|
|
if (externalResults) {
|
|||
|
|
batchResults = externalResults.map((item, index) => normalizeBatchResult(item, index + 1));
|
|||
|
|
} else if (!env.ARK_API_KEY) {
|
|||
|
|
return buildVisionRequestOutput({ workspace, batches, prompt });
|
|||
|
|
} else {
|
|||
|
|
const tasks = batches.map((batch, index) => async () => {
|
|||
|
|
if (index > 0 && batchDelay > 0) await sleep(batchDelay * 1000);
|
|||
|
|
return detectBatch({ batch, batchNo: index + 1, totalBatches: batches.length, env, prompt });
|
|||
|
|
});
|
|||
|
|
batchResults = await runWithConcurrency(tasks, concurrency);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const vehicleInfo = emptyVehicleInfo();
|
|||
|
|
const damages = [];
|
|||
|
|
for (const result of batchResults) {
|
|||
|
|
mergeVehicleInfo(vehicleInfo, result.vehicleInfo);
|
|||
|
|
damages.push(...result.damages);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const outputDoc = {
|
|||
|
|
taskId: workspace.taskId,
|
|||
|
|
workspacePath: workspace.workspacePath,
|
|||
|
|
totalBatches: batches.length,
|
|||
|
|
successBatches: batchResults.filter(item => item.success).length,
|
|||
|
|
damages,
|
|||
|
|
vehicleInfo,
|
|||
|
|
batches: batchResults,
|
|||
|
|
generatedAt: new Date().toISOString(),
|
|||
|
|
};
|
|||
|
|
writeJson(path.join(workspace.workspacePath, 'damages.json'), outputDoc);
|
|||
|
|
emit('detect_damages', '检测完成并写入damages.json', 'completed', { damageCount: damages.length });
|
|||
|
|
return {
|
|||
|
|
success: true,
|
|||
|
|
damagesFound: damages.length,
|
|||
|
|
totalBatches: outputDoc.totalBatches,
|
|||
|
|
successBatches: outputDoc.successBatches,
|
|||
|
|
vehicleInfo,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
(async () => {
|
|||
|
|
try {
|
|||
|
|
process.stdout.write(JSON.stringify(await run(readInput())));
|
|||
|
|
} catch (err) {
|
|||
|
|
process.stdout.write(JSON.stringify({
|
|||
|
|
success: false,
|
|||
|
|
error: err && err.message ? err.message : String(err),
|
|||
|
|
}));
|
|||
|
|
}
|
|||
|
|
})();
|