mfx/tkAiPage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

分包 触发屏幕事件

Browse Source
This commit is contained in:
没复习
2025-08-22 16:59:41 +08:00
parent 5bfb9027b6
commit a8889e1ce6
3 changed files with 247 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
src/composables/useDeviceDiscovery.js Normal file
View File

@@ -0,0 +1,102 @@
// src/composables/useDeviceDiscovery.js
import { nextTick } from 'vue'
/**
* 负责连接 multiplex WebSocket，维护 deviceInformation 列表，
* 并在设备出现时按顺序调用 initCanvas / initVideoStream / getSize。
* 获取设备的ws
*/
export function useDeviceDiscovery({
deviceInformation, // ref([])
initCanvas, // (udid) => void
initVideoStream, // (udid, index) => void
wsActionsRef, // () => wsActions (可能一开始是 null)
td // 你的 useTeardown 实例，可选
}) {
const decoder = new TextDecoder('utf-8')
function open(url, eitwoBuffer) {
const ws = new WebSocket(url)
td && td.setMultiplexWS && td.setMultiplexWS(ws) // 交给 teardown 托管
ws.binaryType = 'arraybuffer'
ws.onopen = () => {
ws.send(eitwoBuffer) // 请求设备列表
}
ws.onmessage = async (event) => {
let data
try {
// 兼容原逻辑：先解码再清理不可见字符再 JSON 解析
data = JSON.parse(decoder.decode(event.data).replace(/[^\x20-\x7F]/g, ''))
} catch (e) {
console.error('[multiplex] parse error:', e)
return
}
if (data.type === 'devicelist') {
// 全量刷新
deviceInformation.value = []
const list = (data.data.list || []).filter(it => it.state === 'device')
for (let i = 0; i < list.length; i++) {
const item = list[i]
deviceInformation.value.push(item)
await nextTick() // 等 v-for 渲染出 <video>
initCanvas(item.udid)
initVideoStream(item.udid, i)
// 延迟拉尺寸（和你原逻辑一致）
setTimeout(() => {
const wsActions = wsActionsRef && wsActionsRef()
wsActions && wsActions.getSize && wsActions.getSize(item.udid, i)
}, 2000)
}
}
else if (data.type === 'device') {
const d = data.data.device
if (d.state === 'offline') {
// 移除离线设备
const idx = deviceInformation.value.findIndex(x => x.udid === d.udid)
if (idx !== -1) {
deviceInformation.value.splice(idx, 1)
}
}
else if (d.state === 'device') {
// 新设备上线
const exists = deviceInformation.value.some(x => x.udid === d.udid)
if (!exists) {
deviceInformation.value.push(d)
// 稍等一会再初始化，保持与原代码行为一致
setTimeout(async () => {
try {
const i = deviceInformation.value.length - 1
await nextTick()
initCanvas(d.udid)
initVideoStream(d.udid, i)
} catch (e) {
console.warn('[multiplex] online init failed:', e)
}
}, 1000)
}
}
}
}
ws.onerror = (e) => {
console.error('[multiplex] ws error:', e)
}
ws.onclose = () => {
// 可按需在这里做重连；现在保持与原逻辑一致不做自动重连
}
return ws
}
return { open }
}

109
src/composables/useTouch.js Normal file
View File

@@ -0,0 +1,109 @@
// src/composables/useTouch.js （纯 JS）
/**
* 提供 sleep / tapAt / longPressAt / drag / clickxy 五个工具。
* clickxy 内会调用 Back、wsActionsRef、setComText、phoneXYinfoRef。
*/
export function useTouch({
deviceInformation, // ref([]) 读取 udid
sendPointer, // 来自 useCanvasPointer
wsActionsRef, // () => wsActions
phoneXYinfoRef, // ref([...]) 内含 id（screen坐标体系的 device id）
Back, // function Back('', index)
setComText, // function setComText(index)
}) {
// 小工具：延迟
const sleep = (ms) => new Promise(r => setTimeout(r, ms))
// 轻点
async function tapAt(udid, index, x, y, holdMs = 80) {
sendPointer(udid, index, 0, x, y) // down
await sleep(holdMs)
sendPointer(udid, index, 1, x, y) // up
}
// 长按
async function longPressAt(udid, index, x, y, holdMs = 1500) {
sendPointer(udid, index, 0, x, y) // down
await sleep(holdMs)
sendPointer(udid, index, 1, x, y) // up
}
// 拖拽/滑动
async function drag(udid, index, x1, y1, x2, y2, durationMs = 300, steps = 8) {
sendPointer(udid, index, 0, x1, y1) // down
const dx = (x2 - x1) / steps
const dy = (y2 - y1) / steps
const dt = Math.max(8, Math.floor(durationMs / steps))
for (let i = 1; i <= steps; i++) {
await sleep(dt)
sendPointer(udid, index, 2, Math.round(x1 + dx * i), Math.round(y1 + dy * i)) // move
}
await sleep(20)
sendPointer(udid, index, 1, x2, y2) // up
}
/**
* clickxy：统一的“坐标点击”入口，保留你原来的 type 语义：
* - type===3：点击 -> 返回 -> 下滑
* - type===2：点击 -> 返回 -> 右滑
* - type===1：点击评论框后粘贴发送（setComText）
* - type===9：长按
* - 默认：普通点击
*/
async function clickxy(x, y, index, type) {
const dev = deviceInformation.value[index]
const wsActions = wsActionsRef && wsActionsRef()
if (!dev || !dev.udid) {
console.error('clickxy: no udid at index', index)
return
}
const udid = dev.udid
try {
if (type === 3) {
await tapAt(udid, index, x, y, 80)
await sleep(300)
Back('', index)
await sleep(300)
wsActions && wsActions.slideDown && wsActions.slideDown(phoneXYinfoRef.value[index].id, index)
return
}
if (type === 2) {
await tapAt(udid, index, x, y, 80)
await sleep(300)
Back('', index)
await sleep(300)
wsActions && wsActions.slideRight && wsActions.slideRight(phoneXYinfoRef.value[index].id, index)
return
}
if (type === 1) {
await tapAt(udid, index, x, y, 80)
await sleep(300)
setComText(index) // 评论框：粘贴发送
return
}
if (type === 9) {
await longPressAt(udid, index, x, y, 1500)
return
}
// 默认：普通轻点
await tapAt(udid, index, x, y, 80)
} catch (e) {
console.error('clickxy error:', e)
}
}
return {
sleep,
tapAt,
longPressAt,
drag,
clickxy,
}
}

222
src/views/VideoStream.vue
View File

@@ -85,8 +85,9 @@ import HostListManagerDialog from '@/components/HostListManagerDialog.vue'
import { useMonitor } from '@/composables/useMonitor'
import { useTeardown } from '@/composables/useTeardown' //销毁
import LeftToolbar from '@/components/LeftToolbar.vue' //左侧工具栏
import { useStreams } from '@/composables/useStreams'
import { useStreams } from '@/composables/useStreams' //视频流处理
import { useDeviceDiscovery } from '@/composables/useDeviceDiscovery' //设备发现
import { useTouch } from '@/composables/useTouch' //触摸事件
const router = useRouter();
let wsActions = null;
let userdata = getUser();
@@ -318,7 +319,21 @@ const buttons = [
}
}
]
//返回上一层
const Back = (udid, index) => {
wslist[index].send(toBufferBtn({ "type": 0, "action": 0, "keycode": 4, "metaState": 0, "repeat": 0 }));
wslist[index].send(toBufferBtn({ "type": 0, "action": 1, "keycode": 4, "metaState": 0, "repeat": 0 }));
}
//返回首页
const Home = (udid, index) => {
wslist[index].send(toBufferBtn({ "type": 0, "action": 0, "keycode": 3, "metaState": 0, "repeat": 0 }));
wslist[index].send(toBufferBtn({ "type": 0, "action": 1, "keycode": 3, "metaState": 0, "repeat": 0 }));
}
//任务视图
const Overview = (udid, index) => {
wslist[index].send(toBufferBtn({ "type": 0, "action": 0, "keycode": 187, "metaState": 0, "repeat": 0 }));
wslist[index].send(toBufferBtn({ "type": 0, "action": 1, "keycode": 187, "metaState": 0, "repeat": 0 }));
}
// 建立 monitor
const {
@@ -347,7 +362,13 @@ const { canvasRef, frameMeta, initCanvas, getCanvasCoordinate, sendPointer } =
toBuffer, // 你已有的工具函数
getWs: (i) => wslist[i] // 取对应 index 的 WebSocket
});
const { sleep, tapAt, longPressAt, drag, clickxy } = useTouch({
deviceInformation,
sendPointer,
wsActionsRef: () => wsActions,
phoneXYinfoRef: phoneXYinfo,
Back,
})
// —— 放在变量区（deviceInformation 已经是 ref([])）——
const pausedDevices = new Set(); // 用 UDID 做键
@@ -379,6 +400,8 @@ const { feedState, pushFrame, resetFeedState, waitForVideoEl, refreshStream } =
openStr,
VideoConverter, // 传入页面已引入的构造器，避免在 composable 重复引入
})
//````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````
// 初始化 手机显示WebSocket 和视频流
const initVideoStream = async (udid, index) => {
@@ -848,7 +871,13 @@ const initVideoStream = async (udid, index) => {
//``````````````````````````````````````````````````````````````````````````````````
};
const { open: openDiscovery } = useDeviceDiscovery({
deviceInformation,
initCanvas,
initVideoStream,
wsActionsRef: () => wsActions,
td
})
// 鼠标按下事件处理
const handleCanvasdown = (udid, e, index) => {
@@ -889,21 +918,7 @@ function alignTo(x, a = ALIGN_BASE, mode = ROUND_MODE) {
/* nearest */ return Math.round(q) * a;
}
//返回上一层
const Back = (udid, index) => {
wslist[index].send(toBufferBtn({ "type": 0, "action": 0, "keycode": 4, "metaState": 0, "repeat": 0 }));
wslist[index].send(toBufferBtn({ "type": 0, "action": 1, "keycode": 4, "metaState": 0, "repeat": 0 }));
}
//返回首页
const Home = (udid, index) => {
wslist[index].send(toBufferBtn({ "type": 0, "action": 0, "keycode": 3, "metaState": 0, "repeat": 0 }));
wslist[index].send(toBufferBtn({ "type": 0, "action": 1, "keycode": 3, "metaState": 0, "repeat": 0 }));
}
//任务视图
const Overview = (udid, index) => {
wslist[index].send(toBufferBtn({ "type": 0, "action": 0, "keycode": 187, "metaState": 0, "repeat": 0 }));
wslist[index].send(toBufferBtn({ "type": 0, "action": 1, "keycode": 187, "metaState": 0, "repeat": 0 }));
}
//回车 index手机下标 state up按下 down抬起 keycode 键
const key = (index, state, keycode) => {
// console.log("退格", index)
@@ -1056,87 +1071,7 @@ onUnmounted(() => {
//获取设备信息
const ObtainDeviceInformation = () => {
// 2. 连接 WebSocket
const ws = new WebSocket("ws://127.0.0.1:8000/?action=multiplex");
td.setMultiplexWS(ws)
ws.binaryType = "arraybuffer";
ws.onopen = () => {
ws.send(eitwo);
};
// 3. 处理接收到的二进制数据
ws.onmessage = async (event) => {
const data = JSON.parse(new TextDecoder('utf-8').decode(event.data).replace(/[^\x20-\x7F]/g, ''));
try {
console.log('数组', data)
if (data.type == "devicelist") {
deviceInformation.value = [];
const filteredList = data.data.list.filter(item => item.state === 'device');
//检测到设备列表时，渲染所有设备
for (let i = 0; i < filteredList.length; i++) {
const item = filteredList[i];
deviceInformation.value.push(item);
await nextTick(); // 等 v-for 渲染出 <video>
initCanvas(item.udid); // 如果它也依赖 DOM，同样要在 nextTick 之后
initVideoStream(item.udid, i); // 直接使用循环变量 i
// getSize 建议放到 wslist[index].onopen 里最稳，
// 若保留延时也可以：
setTimeout(() => wsActions?.getSize(item.udid, i), 2000); // 直接使用循环变量 i
}
} else if (data.type == "device") {
if (data.data.device.state === "offline") {
//监听设备信息，出现离线设备，则删除设备信息
deviceInformation.value.forEach((item, index) => {
if (item.udid === data.data.device.udid) {
deviceInformation.value.splice(index, 1);
if (index < wslist.length - 1) {
deviceInformation.value.forEach((item, index1) => {
//关闭websocket连接
wslist.forEach((item, index) => {
new VideoConvertitem.close();
})
//重新连接websocket
new Promise((resolve, reject) => {
setTimeout(() => {
try {
initVideoStream(item.udid, index1);
initCanvas(item.udid);
} catch (error) {
reject(error);
}
}, 300);
});
})
}
console.log("deviceInformation", deviceInformation.value);
}
})
} else if (data.data.device.state === "device") {
let isrepeat = false;
//监听设备信息，出现新设备，则添加设备信息
deviceInformation.value.forEach((item, index) => {
if (item.udid == data.data.device.udid) {
isrepeat = true;
}
})
if (!isrepeat) {
deviceInformation.value.push(data.data.device);
new Promise((resolve, reject) => {
setTimeout(() => {
try {
initVideoStream(data.data.device.udid, deviceInformation.value.length - 1);
initCanvas(data.data.device.udid);
} catch (error) {
// reject(error);
}
}, 1000);
});
}
};
}
} catch (e) {
console.error(e);
}
};
openDiscovery("ws://127.0.0.1:8000/?action=multiplex", eitwo)
}
//喜欢-输入评论
@@ -1259,91 +1194,6 @@ function getComArr(index, type) {
// 小工具
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
async function tapAt(udid, index, x, y, holdMs = 80) {
sendPointer(udid, index, 0, x, y); // down
await sleep(holdMs);
sendPointer(udid, index, 1, x, y); // up
}
async function longPressAt(udid, index, x, y, holdMs = 1500) {
sendPointer(udid, index, 0, x, y); // down
await sleep(holdMs);
sendPointer(udid, index, 1, x, y); // up
}
/** 可用于自定义滑动/拖拽（如果以后要替代 slideDown/slideRight） */
async function drag(udid, index, x1, y1, x2, y2, durationMs = 300, steps = 8) {
sendPointer(udid, index, 0, x1, y1); // down
const dx = (x2 - x1) / steps;
const dy = (y2 - y1) / steps;
const dt = Math.max(8, Math.floor(durationMs / steps));
for (let i = 1; i <= steps; i++) {
await sleep(dt);
sendPointer(udid, index, 2, Math.round(x1 + dx * i), Math.round(y1 + dy * i)); // move
}
await sleep(20);
sendPointer(udid, index, 1, x2, y2); // up
}
// 用 pointer down/up/move 改写后的 clickxy
async function clickxy(x, y, index, type) {
const udid = deviceInformation.value[index].udid;
if (!udid) {
console.error('clickxy: no udid');
return;
};
try {
if (type === 3) {
// 关注/私信后的返回和下滑
await tapAt(udid, index, x, y, 80);
await sleep(300);
Back('', index);
await sleep(300);
wsActions.slideDown(phoneXYinfo.value[index].id, index);
// 你原代码里这里还有 start(...)，如果没定义会报错；需要的话保留：
// await sleep(300);
// start(phoneXYinfo.value[index].id, index);
return;
}
if (type === 2) {
// 评论后的返回和右滑
await tapAt(udid, index, x, y, 80);
await sleep(300);
Back('', index);
await sleep(300);
wsActions.slideRight(phoneXYinfo.value[index].id, index);
return;
}
if (type === 1) {
// 评论框点击 -> 粘贴发送
await tapAt(udid, index, x, y, 80);
await sleep(300);
setComText(index);
return;
}
if (type === 9) {
// 长按
await longPressAt(udid, index, x, y, 1500);
return;
}
// 默认：普通轻点
await tapAt(udid, index, x, y, 80);
} catch (e) {
console.error('clickxy error:', e);
}
}
const reload = (opts = {}) => {
const { onlySelected = false, hard = true } = opts;
const targets = (onlySelected && selectedDevice.value !== 999)