element-call
element-call copied to clipboard
Published
1 month ago
•
element-hq
element-hq
Not connecting to live kit (Webview)
Steps to reproduce
1. Where are you starting? What can you see? I've prepared a simple html for rendering a widget inside a webview. It works normally for lobby, but as soon as I try to join the call the live kit connection doesn't seem to properly initialize
<!DOCTYPE html>
<html style="background-color: transparent; height: 100%; width: 100%; margin: 0px; padding: 0px">
<head>
<meta charset="utf-8" />
<meta http-equiv="content-type" content="text/html;charset=utf-8" />
<meta name="viewport" content="initial-scale=1.0;maximum-scale=1.0" />
</head>
<body style="background-color: transparent; height: 100%; display: flex; justify-content: center; align-items: center; width: 100%; margin: 0px; padding: 0px">
<script type="text/javascript" src="https://unpkg.com/[email protected]/dist/api.js"></script>
<script type="text/javascript">
var widgetApi;
const iframeSrc = "__IFRAME_SRC__";
const roomId = "__ROOM_ID__";
const userId = "__USER_ID__";
const deviceId = "__DEVICE_ID__";
const widgetId = "__WIDGET_ID__";
const homeServerUrl = "__BASE_URL__";
const parentUrl = "__PARENT_URL__";
const memberPayload = `__MEMBER_PAYLOAD__`;
const callPayload = `__CALL_PAYLOAD__`;
const callMemberPayload = `__CALL_MEMBER_PAYLOAD__`;
const createRoomPayload = `__CREATE_ROOM_PAYLOAD__`;
const widgetToken = "__WIDGET_TOKEN__";
const GROUP_CALL_EVENT = "org.matrix.msc3401.call";
const GROUP_CALL_MEMBER_EVENT = "org.matrix.msc3401.call.member";
class CallWidgetDriver extends (window.mxwidgets?.WidgetDriver || WidgetDriver) {
validateCapabilities(requested) {
return Promise.resolve(requested);
}
async sendEvent(eventType, content, stateKey = null, roomId = null) {
console.log("sendEvent", { eventType, content, stateKey, roomId });
// Use REST API to send Matrix state events
const matrixUrl = homeServerUrl + "/_matrix/client/v3/rooms/" + encodeURIComponent(roomId) + "/state/" + encodeURIComponent(eventType) + "/" + encodeURIComponent(stateKey);
const response = await fetch(matrixUrl, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
'Authorization': "Bearer " + widgetToken,
},
body: JSON.stringify(content),
})
const data = await response.json();
console.log('Matrix event sent:', data);
return Promise.resolve({
roomId,
eventId: data.event_id,
});
}
async sendToDevice(eventType, encrypted, contentMap) {
console.log("sendToDevice", { eventType, encrypted, contentMap });
const txnId = "txn" + Math.round(Math.random() * 1000000);
const matrixUrl = homeServerUrl + "/_matrix/client/v3/sendToDevice/" + encodeURIComponent(eventType) + "/" + txnId;
const response = await fetch(matrixUrl, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
'Authorization': "Bearer " + widgetToken,
},
body: JSON.stringify(contentMap),
})
const data = await response.json();
console.log('To-device event sent:', data);
return Promise.resolve();
}
readStateEvents(eventType, stateKey, limit, roomIds = null) {
let stateEvents = [];
if (eventType === "m.room.member") {
stateEvents = JSON.parse(memberPayload);
} else if (eventType === GROUP_CALL_EVENT) {
stateEvents = JSON.parse(callPayload);
} else if (eventType === GROUP_CALL_MEMBER_EVENT) {
stateEvents = JSON.parse(callMemberPayload);
} else if (eventType === "m.room.create") {
stateEvents = JSON.parse(createRoomPayload);
}
console.log("readStateEvents", { eventType, stateKey, limit, roomIds }, stateEvents);
return Promise.resolve(stateEvents);
}
readRoomState(roomId, eventType, stateKey) {
return this.readStateEvents(eventType, stateKey, Number.MAX_SAFE_INTEGER, [roomId]);
}
async *getTurnServers() {
yield {
uris: ["stun:turn.matrix.org"],
username: "",
password: "",
};
}
async sendDelayedEvent(delay, parentDelayId, eventType, content, stateKey = null, roomId = null) {
console.log("sendDelayedEvent", { delay, parentDelayId, eventType, content, stateKey, roomId });
const matrixUrl = homeServerUrl + "/_matrix/client/v3/rooms/" + encodeURIComponent(roomId) + "/state/" + encodeURIComponent(eventType) + "/" + encodeURIComponent(stateKey) + "?org.matrix.msc4140.delay=" + encodeURIComponent(delay);
const response = await fetch(matrixUrl, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
'Authorization': "Bearer " + widgetToken,
},
body: JSON.stringify(content),
});
const data = await response.json();
console.log('Matrix delayed event sent:', data);
return Promise.resolve({
roomId,
delayId: data.delay_id,
});
}
async updateDelayedEvent(delayId, action) {
console.log("updateDelayedEvent", { delayId, action });
// Use REST API to update/cancel delayed event
const matrixUrl = homeServerUrl + "/_matrix/client/unstable/org.matrix.msc4140/delayed_events/" + encodeURIComponent(delayId);
const response = await fetch(matrixUrl, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': "Bearer " + widgetToken,
},
body: JSON.stringify({ action }),
});
const data = await response.json();
console.log('Matrix delayed event updated:', data);
return Promise.resolve();
}
readRoomAccountData(eventType, roomIds = null) {
console.log("readRoomAccountData", { eventType, roomIds });
return Promise.resolve([]);
}
readRoomEvents(eventType, msgtype, limit, roomIds = null, since = null) {
console.log("readRoomEvents", { eventType, msgtype, limit, roomIds, since });
return Promise.resolve([]);
}
readEventRelations(eventId, roomId, relationType, eventType, from, to, limit, direction) {
console.log("readEventRelations", { eventId, roomId, relationType, eventType, from, to, limit, direction });
return Promise.resolve({ chunk: [] });
}
askOpenID(observer) {
console.log("askOpenID", { observer });
observer.update({state: "blocked" });
}
}
const onHangup = ev => {
console.log("onHangup");
ev.preventDefault();
widgetApi.transport.reply(ev.detail, {});
widgetApi.off("action:im.vector.hangup", onHangup);
widgetApi.removeAllListeners();
widgetApi.stop();
};
const onReady = async () => {
console.log("onReady");
widgetApi.on("action:im.vector.hangup", onHangup);
};
const onIframeLoad = () => {
const callWidget = {
id: widgetId,
creatorUserId: userId,
type: "m.custom",
url: iframeSrc,
origin: parentUrl,
waitForIframeLoad: false,
};
const widget = new mxwidgets.Widget(callWidget);
const callWidgetIframe = document.getElementById("call-widget-iframe");
callWidgetIframe.src = iframeSrc;
callWidgetIframe.onload = "";
const widgetDriver = new CallWidgetDriver();
widgetApi = new mxwidgets.ClientWidgetApi(widget, callWidgetIframe, widgetDriver);
widgetApi.once("ready", onReady);
};
</script>
<iframe
id="call-widget-iframe"
style="height: 100%; width: 100%; border-width: 0px; border-radius: 0px;"
allow="camera;microphone"
onload="onIframeLoad()"
>
</iframe>
</body>
</html>
2. What do you click? Click join a call
Outcome
What did you expect?
Can see video/audio stream of current user and other patricipants
What happened instead?
Blank screen
Operating system
Android
Browser information
Webview sandbox
URL for webapp
No response
Will you send logs?
Yes
logs
The resource https://selfhosted.element.app/config.json was preloaded using link preload but not used within a few seconds from the window's load event. Please make sure it has an appropriate `as` value and it is preloaded intentionally.
[MatrixRTCSession !PUIQTHpGvPFxoeyJxH:selfhosted.element.app] Using to-device with room fallback transport for encryption keys
rtcSessionHelpers.ts:143 [PostmessageTransport] Sending object to https://selfhosted.element.app: {api: 'fromWidget', widgetId: '1df94dc3-1bc9-4489-9989-77a205a4b382', requestId: 'widgetapi-1751372923113', action: 'io.element.join', data: {…}}
rageshake.ts:495 [LivekitRoom] Create LiveKit room
rageshake.ts:495 Created ExternalE2EEKeyProvider (shared key)
rageshake.ts:495 setting up e2ee
rageshake.ts:495 initializing worker {worker: Worker}
rageshake.ts:495 [Lifecycle] GroupCallView Component unmounted
rageshake.ts:495 [Lifecycle] InCallView Component mounted, livekitroom state disconnected
GroupCallView.tsx:374 [PostmessageTransport] Sending object to https://selfhosted.element.app: {api: 'fromWidget', widgetId: '1df94dc3-1bc9-4489-9989-77a205a4b382', requestId: 'widgetapi-1751372923248', action: 'set_always_on_screen', data: {…}}
rageshake.ts:495 Session in room !PUIQTHpGvPFxoeyJxH:selfhosted.element.app changed to joined
InCallView.tsx:404 [PostmessageTransport] Sending object to https://selfhosted.element.app: {api: 'fromWidget', widgetId: '1df94dc3-1bc9-4489-9989-77a205a4b382', requestId: 'widgetapi-1751372923526', action: 'io.element.spotlight_layout', data: {…}}
rageshake.ts:495 Could not set audio output: cannot switch audio output, setSinkId not supported {room: '', roomID: undefined, participant: '', pID: ''}
PostmessageTransport.js:93 [PostmessageTransport] Sending object to https://selfhosted.element.app: {api: 'fromWidget', widgetId: '1df94dc3-1bc9-4489-9989-77a205a4b382', requestId: 'widgetapi-1751372923113', action: 'io.element.join', data: {…}, …}
PostmessageTransport.js:93 [PostmessageTransport] Sending object to https://selfhosted.element.app: {api: 'fromWidget', widgetId: '1df94dc3-1bc9-4489-9989-77a205a4b382', requestId: 'widgetapi-1751372923248', action: 'set_always_on_screen', data: {…}, …}
PostmessageTransport.js:93 [PostmessageTransport] Sending object to https://selfhosted.element.app: {api: 'fromWidget', widgetId: '1df94dc3-1bc9-4489-9989-77a205a4b382', requestId: 'widgetapi-1751372923526', action: 'io.element.spotlight_layout', data: {…}, …}
rageshake.ts:495 Failed to send join action uC: Unknown or unsupported action: io.element.join
at v0e.handleResponse (https://selfhosted.element.app/assets/index-Cv6uhLte.js:26:64308)
at v0e.handleMessage (https://selfhosted.element.app/assets/index-Cv6uhLte.js:26:63873)
at https://selfhosted.element.app/assets/index-Cv6uhLte.js:26:63553
rageshake.ts:495 Error calling setAlwaysOnScreen(true) uC: Unknown or unsupported action: set_always_on_screen
at v0e.handleResponse (https://selfhosted.element.app/assets/index-Cv6uhLte.js:26:64308)
at v0e.handleMessage (https://selfhosted.element.app/assets/index-Cv6uhLte.js:26:63873)
at https://selfhosted.element.app/assets/index-Cv6uhLte.js:26:63553
rageshake.ts:495 Failed to send layout change to widget API uC: Unknown or unsupported action: io.element.spotlight_layout
at v0e.handleResponse (https://selfhosted.element.app/assets/index-Cv6uhLte.js:26:64308)
at v0e.handleMessage (https://selfhosted.element.app/assets/index-Cv6uhLte.js:26:63873)
at https://selfhosted.element.app/assets/index-Cv6uhLte.js:26:63553
livekit-client.e2ee.worker-ChP6sqnt.js:1 worker initialized
livekit-client.e2ee.worker-ChP6sqnt.js:1 set shared key {index: undefined}
PostmessageTransport.ts:86 [PostmessageTransport] Sending object to https://selfhosted.element.app: {api: 'fromWidget', widgetId: '1df94dc3-1bc9-4489-9989-77a205a4b382', requestId: 'widgetapi-1751372924100', action: 'send_event', data: {…}}
(index):99 sendDelayedEvent {delay: 8000, parentDelayId: null, eventType: 'org.matrix.msc3401.call.member', content: {…}, stateKey: '_@userId003:selfhosted.element.app_7a57f41b-3507-468d-b47c-22389553ce39', …}
(index):110 Matrix delayed event sent: {delay_id: 'syd_PVRCOMMjabJnfsgaZupn'}
PostmessageTransport.js:93 [PostmessageTransport] Sending object to https://selfhosted.element.app: {api: 'fromWidget', widgetId: '1df94dc3-1bc9-4489-9989-77a205a4b382', requestId: 'widgetapi-1751372924100', action: 'send_event', data: {…}, …}
PostmessageTransport.ts:86 [PostmessageTransport] Sending object to https://selfhosted.element.app: {api: 'fromWidget', widgetId: '1df94dc3-1bc9-4489-9989-77a205a4b382', requestId: 'widgetapi-1751372924203', action: 'send_event', data: {…}}
(index):35 sendEvent {eventType: 'org.matrix.msc3401.call.member', content: {…}, stateKey: '_@userId003:selfhosted.element.app_7a57f41b-3507-468d-b47c-22389553ce39', roomId: '!PUIQTHpGvPFxoeyJxH:selfhosted.element.app'}
(index):47 Matrix event sent: {event_id: '$7Lli2wDM6cw1_4iOTn_fvPaptgwnY8k7NF7qmzH7FDU'}
PostmessageTransport.js:93 [PostmessageTransport] Sending object to https://selfhosted.element.app: {api: 'fromWidget', widgetId: '1df94dc3-1bc9-4489-9989-77a205a4b382', requestId: 'widgetapi-1751372924203', action: 'send_event', data: {…}, …}
PostmessageTransport.ts:86 [PostmessageTransport] Sending object to https://selfhosted.element.app: {api: 'fromWidget', widgetId: '1df94dc3-1bc9-4489-9989-77a205a4b382', requestId: 'widgetapi-1751372924438', action: 'org.matrix.msc4157.update_delayed_event', data: {…}}
(index):118 updateDelayedEvent {delayId: 'syd_PVRCOMMjabJnfsgaZupn', action: 'restart'}
(index):130 Matrix delayed event updated: {}
PostmessageTransport.js:93 [PostmessageTransport] Sending object to https://selfhosted.element.app: {api: 'fromWidget', widgetId: '1df94dc3-1bc9-4489-9989-77a205a4b382', requestId: 'widgetapi-1751372924438', action: 'org.matrix.msc4157.update_delayed_event', data: {…}, …}
rageshake.ts:495 Sent updated call member event.
... then it goes back and forth updating delayed event...
@mihamor Did you end up getting this to work, I have a similar use case, can't seem to get it working it's stuck on a loading screen till the call ends
Can you give a more detailed explanation what you are trying to do? How is it supposed to work?
On first sight I dont see the branch where the widget gets synced events (It needs to get the synced state events to determine the livekit SFU jwt service)
@toger5 @binatari @fkwp
hey, so I ended up creating my own custom RN <-> webview bridge for processing WidgetDriver methods
...
const requestResolvers = {};
function generateRequestId() {
return "req_" + String(Math.random()).slice(8);
}
const REQUEST_TIMEOUT_MS = 10000;
function createAppRequest(requestType, payload) {
const requestId = generateRequestId();
const finalPayload = {
...payload,
type: requestType,
requestId,
};
window.ReactNativeWebView.postMessage(JSON.stringify(finalPayload));
return new Promise((resolve, reject) => {
const timeoutId = setTimeout(() => {
if (requestResolvers[requestId]) {
reject(new Error('Request timed out ' + requestId));
delete requestResolvers[requestId];
}
}, REQUEST_TIMEOUT_MS);
requestResolvers[requestId] = { resolve, reject, timeoutId };
});
}
function handleAppResponse(event) {
// ...
}
And in WidgetDriver:
readStateEvents(eventType, stateKey, limit, roomIds = null) {
return createAppRequest("readStateEvents", {
eventType,
stateKey,
limit,
roomIds,
});
}
And with that I was able to finally see my own user in the room, but only if I post a new org.matrix.msc3401.call.member event beforehand.
But I still wasn't able to see new participants joining and theirs video/audio streams.
On first sight I dont see the branch where the widget gets synced events (It needs to get the synced state events to determine the livekit SFU jwt service)
I suspect that the widget state should be somehow synced continuously, but not sure which WidgetDriver method is responsible for that as readStateEvents is only called once in a lobby for me.
@binatari Great, One consideration i have in my own implementation is if the calls will work when the app is killed.
Start element call in Element web and search for Transport there will be the widget API transport logs that will make things a lot clearer. Those widget actions need to be sent by your widget driver.
All this code should be available in the element web repok in the stop gap widget driver class. Looking there should allow you to copy paste a lot of the work