Merge branch 'main' into audio-observability

src/otel/OTelCall.ts

@@ -0,0 +1,119 @@
+/*
+Copyright 2023 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { Span } from "@opentelemetry/api";
+import { MatrixCall } from "matrix-js-sdk";
+import { CallEvent } from "matrix-js-sdk/src/webrtc/call";
+
+import { ObjectFlattener } from "./ObjectFlattener";
+
+/**
+ * Tracks an individual call within a group call, either to a full-mesh peer or a focus
+ */
+export class OTelCall {
+  constructor(
+    public userId: string,
+    public deviceId: string,
+    public call: MatrixCall,
+    public span: Span
+  ) {
+    if (call.peerConn) {
+      this.addCallPeerConnListeners();
+    } else {
+      this.call.once(
+        CallEvent.PeerConnectionCreated,
+        this.addCallPeerConnListeners
+      );
+    }
+  }
+
+  public dispose() {
+    this.call.peerConn.removeEventListener(
+      "connectionstatechange",
+      this.onCallConnectionStateChanged
+    );
+    this.call.peerConn.removeEventListener(
+      "signalingstatechange",
+      this.onCallSignalingStateChanged
+    );
+    this.call.peerConn.removeEventListener(
+      "iceconnectionstatechange",
+      this.onIceConnectionStateChanged
+    );
+    this.call.peerConn.removeEventListener(
+      "icegatheringstatechange",
+      this.onIceGatheringStateChanged
+    );
+    this.call.peerConn.removeEventListener(
+      "icecandidateerror",
+      this.onIceCandidateError
+    );
+  }
+
+  private addCallPeerConnListeners = (): void => {
+    this.call.peerConn.addEventListener(
+      "connectionstatechange",
+      this.onCallConnectionStateChanged
+    );
+    this.call.peerConn.addEventListener(
+      "signalingstatechange",
+      this.onCallSignalingStateChanged
+    );
+    this.call.peerConn.addEventListener(
+      "iceconnectionstatechange",
+      this.onIceConnectionStateChanged
+    );
+    this.call.peerConn.addEventListener(
+      "icegatheringstatechange",
+      this.onIceGatheringStateChanged
+    );
+    this.call.peerConn.addEventListener(
+      "icecandidateerror",
+      this.onIceCandidateError
+    );
+  };
+
+  public onCallConnectionStateChanged = (): void => {
+    this.span.addEvent("matrix.call.callConnectionStateChange", {
+      callConnectionState: this.call.peerConn.connectionState,
+    });
+  };
+
+  public onCallSignalingStateChanged = (): void => {
+    this.span.addEvent("matrix.call.callSignalingStateChange", {
+      callSignalingState: this.call.peerConn.signalingState,
+    });
+  };
+
+  public onIceConnectionStateChanged = (): void => {
+    this.span.addEvent("matrix.call.iceConnectionStateChange", {
+      iceConnectionState: this.call.peerConn.iceConnectionState,
+    });
+  };
+
+  public onIceGatheringStateChanged = (): void => {
+    this.span.addEvent("matrix.call.iceGatheringStateChange", {
+      iceGatheringState: this.call.peerConn.iceGatheringState,
+    });
+  };
+
+  public onIceCandidateError = (ev: Event): void => {
+    const flatObject = {};
+    ObjectFlattener.flattenObjectRecursive(ev, flatObject, "error.", 0);
+
+    this.span.addEvent("matrix.call.iceCandidateError", flatObject);
+  };
+}

src/otel/OTelGroupCallMembership.ts

@@ -15,16 +15,35 @@ limitations under the License.
 */
 
 import opentelemetry, { Span, Attributes, Context } from "@opentelemetry/api";
-import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
 import {
   GroupCall,
   MatrixClient,
   MatrixEvent,
   RoomMember,
 } from "matrix-js-sdk";
-import { VoipEvent } from "matrix-js-sdk/src/webrtc/call";
+import { logger } from "matrix-js-sdk/src/logger";
+import {
+  CallError,
+  CallState,
+  MatrixCall,
+  VoipEvent,
+} from "matrix-js-sdk/src/webrtc/call";
+import {
+  CallsByUserAndDevice,
+  GroupCallError,
+  GroupCallEvent,
+  GroupCallStatsReport,
+} from "matrix-js-sdk/src/webrtc/groupCall";
+import {
+  ConnectionStatsReport,
+  ByteSentStatsReport,
+  SummaryStatsReport,
+} from "matrix-js-sdk/src/webrtc/stats/statsReport";
+import { setSpan } from "@opentelemetry/api/build/esm/trace/context-utils";
 
 import { ElementCallOpenTelemetry } from "./otel";
+import { ObjectFlattener } from "./ObjectFlattener";
+import { OTelCall } from "./OTelCall";
 
 /**
  * Flattens out an object into a single layer with components
@@ -55,7 +74,7 @@ function flattenVoipEventRecursive(
     );
 
   for (const [k, v] of Object.entries(obj)) {
-    if (["string", "number"].includes(typeof v)) {
+    if (["string", "number", "boolean"].includes(typeof v)) {
       flatObject[prefix + k] = v;
     } else if (typeof v === "object") {
       flattenVoipEventRecursive(
@@ -73,21 +92,41 @@ function flattenVoipEventRecursive(
  */
 export class OTelGroupCallMembership {
   private callMembershipSpan?: Span;
-  private callMembershipContext?: Context;
-  private myUserId: string;
-  private myMember: RoomMember;
+  private groupCallContext?: Context;
+  private myUserId = "unknown";
+  private myDeviceId: string;
+  private myMember?: RoomMember;
+  private callsByCallId = new Map<string, OTelCall>();
+  private statsReportSpan: {
+    span: Span | undefined;
+    stats: OTelStatsReportEvent[];
+  };
   private readonly speakingSpans = new Map<RoomMember, Map<string, Span>>();
 
   constructor(private groupCall: GroupCall, client: MatrixClient) {
-    this.myUserId = client.getUserId();
-    this.myMember = groupCall.room.getMember(client.getUserId());
+    const clientId = client.getUserId();
+    if (clientId) {
+      this.myUserId = clientId;
+      const myMember = groupCall.room.getMember(clientId);
+      if (myMember) {
+        this.myMember = myMember;
+      }
+    }
+    this.myDeviceId = client.getDeviceId() || "unknown";
+    this.statsReportSpan = { span: undefined, stats: [] };
+    this.groupCall.on(GroupCallEvent.CallsChanged, this.onCallsChanged);
+  }
 
-    ElementCallOpenTelemetry.instance.provider.resource.attributes[
-      SemanticResourceAttributes.SERVICE_NAME
-    ] = `element-call-${this.myUserId}-${client.getDeviceId()}`;
+  dispose() {
+    this.groupCall.removeListener(
+      GroupCallEvent.CallsChanged,
+      this.onCallsChanged
+    );
   }
 
   public onJoinCall() {
+    if (!ElementCallOpenTelemetry.instance) return;
+
     // Create the main span that tracks the time we intend to be in the call
     this.callMembershipSpan =
       ElementCallOpenTelemetry.instance.tracer.startSpan(
@@ -98,12 +137,13 @@ export class OTelGroupCallMembership {
       this.groupCall.groupCallId
     );
     this.callMembershipSpan.setAttribute("matrix.userId", this.myUserId);
+    this.callMembershipSpan.setAttribute("matrix.deviceId", this.myDeviceId);
     this.callMembershipSpan.setAttribute(
       "matrix.displayName",
-      this.myMember.name
+      this.myMember ? this.myMember.name : "unknown-name"
     );
 
-    this.callMembershipContext = opentelemetry.trace.setSpan(
+    this.groupCallContext = opentelemetry.trace.setSpan(
       opentelemetry.context.active(),
       this.callMembershipSpan
     );
@@ -116,7 +156,7 @@ export class OTelGroupCallMembership {
     // and end the span to indicate we've left
     this.callMembershipSpan!.end();
     this.callMembershipSpan = undefined;
-    this.callMembershipContext = undefined;
+    this.groupCallContext = undefined;
   }
 
   public onUpdateRoomState(event: MatrixEvent) {
@@ -129,28 +169,111 @@ export class OTelGroupCallMembership {
     }
 
     this.callMembershipSpan?.addEvent(
-      `otel_onRoomStateEvent_${event.getType()}`,
+      `matrix.roomStateEvent_${event.getType()}`,
       flattenVoipEvent(event.getContent())
     );
   }
 
-  public onSendEvent(event: VoipEvent) {
+  public onCallsChanged = (calls: CallsByUserAndDevice) => {
+    for (const [userId, userCalls] of calls.entries()) {
+      for (const [deviceId, call] of userCalls.entries()) {
+        if (!this.callsByCallId.has(call.callId)) {
+          if (ElementCallOpenTelemetry.instance) {
+            const span = ElementCallOpenTelemetry.instance.tracer.startSpan(
+              `matrix.call`,
+              undefined,
+              this.groupCallContext
+            );
+            // XXX: anonymity
+            span.setAttribute("matrix.call.target.userId", userId);
+            span.setAttribute("matrix.call.target.deviceId", deviceId);
+            const displayName =
+              this.groupCall.room.getMember(userId)?.name ?? "unknown";
+            span.setAttribute("matrix.call.target.displayName", displayName);
+            this.callsByCallId.set(
+              call.callId,
+              new OTelCall(userId, deviceId, call, span)
+            );
+          }
+        }
+      }
+    }
+
+    for (const callTrackingInfo of this.callsByCallId.values()) {
+      const userCalls = calls.get(callTrackingInfo.userId);
+      if (!userCalls || !userCalls.has(callTrackingInfo.deviceId)) {
+        callTrackingInfo.span.end();
+        this.callsByCallId.delete(callTrackingInfo.call.callId);
+      }
+    }
+  };
+
+  public onCallStateChange(call: MatrixCall, newState: CallState) {
+    const callTrackingInfo = this.callsByCallId.get(call.callId);
+    if (!callTrackingInfo) {
+      logger.error(`Got call state change for unknown call ID ${call.callId}`);
+      return;
+    }
+
+    callTrackingInfo.span.addEvent("matrix.call.stateChange", {
+      state: newState,
+    });
+  }
+
+  public onSendEvent(call: MatrixCall, event: VoipEvent) {
     const eventType = event.eventType as string;
     if (!eventType.startsWith("m.call")) return;
 
+    const callTrackingInfo = this.callsByCallId.get(call.callId);
+    if (!callTrackingInfo) {
+      logger.error(`Got call send event for unknown call ID ${call.callId}`);
+      return;
+    }
+
     if (event.type === "toDevice") {
-      this.callMembershipSpan?.addEvent(
+      callTrackingInfo.span.addEvent(
         `matrix.sendToDeviceEvent_${event.eventType}`,
         flattenVoipEvent(event)
       );
     } else if (event.type === "sendEvent") {
-      this.callMembershipSpan?.addEvent(
+      callTrackingInfo.span.addEvent(
         `matrix.sendToRoomEvent_${event.eventType}`,
         flattenVoipEvent(event)
       );
     }
   }
 
+  public onReceivedVoipEvent(event: MatrixEvent) {
+    // These come straight from CallEventHandler so don't have
+    // a call already associated (in principle we could receive
+    // events for calls we don't know about).
+    const callId = event.getContent().call_id;
+    if (!callId) {
+      this.callMembershipSpan?.addEvent("matrix.receive_voip_event_no_callid", {
+        "sender.userId": event.getSender(),
+      });
+      logger.error("Received call event with no call ID!");
+      return;
+    }
+
+    const call = this.callsByCallId.get(callId);
+    if (!call) {
+      this.callMembershipSpan?.addEvent(
+        "matrix.receive_voip_event_unknown_callid",
+        {
+          "sender.userId": event.getSender(),
+        }
+      );
+      logger.error("Received call event for unknown call ID " + callId);
+      return;
+    }
+
+    call.span.addEvent("matrix.receive_voip_event", {
+      "sender.userId": event.getSender(),
+      ...flattenVoipEvent(event.getContent()),
+    });
+  }
+
   public onToggleMicrophoneMuted(newValue: boolean) {
     this.callMembershipSpan?.addEvent("matrix.toggleMicMuted", {
       "matrix.microphone.muted": newValue,
@@ -194,7 +317,7 @@ export class OTelGroupCallMembership {
         const span = ElementCallOpenTelemetry.instance.tracer.startSpan(
           "matrix.audioActivity",
           undefined,
-          this.callMembershipContext
+          this.groupCallContext
         );
         span.setAttribute("matrix.userId", member.userId);
         span.setAttribute("matrix.displayName", member.rawDisplayName);
@@ -210,4 +333,106 @@ export class OTelGroupCallMembership {
       if (deviceMap?.size === 0) this.speakingSpans.delete(member);
     }
   }
+
+  public onCallError(error: CallError, call: MatrixCall) {
+    const callTrackingInfo = this.callsByCallId.get(call.callId);
+    if (!callTrackingInfo) {
+      logger.error(`Got error for unknown call ID ${call.callId}`);
+      return;
+    }
+
+    callTrackingInfo.span.recordException(error);
+  }
+
+  public onGroupCallError(error: GroupCallError) {
+    this.callMembershipSpan?.recordException(error);
+  }
+
+  public onUndecryptableToDevice(event: MatrixEvent) {
+    this.callMembershipSpan?.addEvent("matrix.toDevice.undecryptable", {
+      "sender.userId": event.getSender(),
+    });
+  }
+
+  public onConnectionStatsReport(
+    statsReport: GroupCallStatsReport<ConnectionStatsReport>
+  ) {
+    if (!ElementCallOpenTelemetry.instance) return;
+
+    const type = OTelStatsReportType.ConnectionReport;
+    const data =
+      ObjectFlattener.flattenConnectionStatsReportObject(statsReport);
+    this.buildStatsEventSpan({ type, data });
+  }
+
+  public onByteSentStatsReport(
+    statsReport: GroupCallStatsReport<ByteSentStatsReport>
+  ) {
+    if (!ElementCallOpenTelemetry.instance) return;
+
+    const type = OTelStatsReportType.ByteSentReport;
+    const data = ObjectFlattener.flattenByteSentStatsReportObject(statsReport);
+    this.buildStatsEventSpan({ type, data });
+  }
+
+  public onSummaryStatsReport(
+    statsReport: GroupCallStatsReport<SummaryStatsReport>
+  ) {
+    if (!ElementCallOpenTelemetry.instance) return;
+
+    const type = OTelStatsReportType.SummaryReport;
+    const data = ObjectFlattener.flattenSummaryStatsReportObject(statsReport);
+    this.buildStatsEventSpan({ type, data });
+  }
+
+  private buildStatsEventSpan(event: OTelStatsReportEvent): void {
+    // @ TODO: fix this - Because on multiple calls we receive multiple stats report spans.
+    // This could be break if stats arrived in same time from different call objects.
+    if (this.statsReportSpan.span === undefined && this.callMembershipSpan) {
+      const ctx = setSpan(
+        opentelemetry.context.active(),
+        this.callMembershipSpan
+      );
+      this.statsReportSpan.span =
+        ElementCallOpenTelemetry.instance.tracer.startSpan(
+          "matrix.groupCallMembership.statsReport",
+          undefined,
+          ctx
+        );
+      this.statsReportSpan.span.setAttribute(
+        "matrix.confId",
+        this.groupCall.groupCallId
+      );
+      this.statsReportSpan.span.setAttribute("matrix.userId", this.myUserId);
+      this.statsReportSpan.span.setAttribute(
+        "matrix.displayName",
+        this.myMember ? this.myMember.name : "unknown-name"
+      );
+
+      this.statsReportSpan.span.addEvent(event.type, event.data);
+      this.statsReportSpan.stats.push(event);
+    } else if (
+      this.statsReportSpan.span !== undefined &&
+      this.callMembershipSpan
+    ) {
+      this.statsReportSpan.span.addEvent(event.type, event.data);
+      this.statsReportSpan.stats.push(event);
+      // if received all three types of stats close this
+      if (this.statsReportSpan.stats.length === 3) {
+        this.statsReportSpan.span.end();
+        this.statsReportSpan = { span: undefined, stats: [] };
+      }
+    }
+  }
+}
+
+interface OTelStatsReportEvent {
+  type: OTelStatsReportType;
+  data: Attributes;
+}
+
+enum OTelStatsReportType {
+  ConnectionReport = "matrix.stats.connection",
+  ByteSentReport = "matrix.stats.byteSent",
+  SummaryReport = "matrix.stats.summary",
 }

src/otel/ObjectFlattener.ts

@@ -0,0 +1,97 @@
+/*
+Copyright 2023 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import { Attributes } from "@opentelemetry/api";
+import { GroupCallStatsReport } from "matrix-js-sdk/src/webrtc/groupCall";
+import {
+  ByteSentStatsReport,
+  ConnectionStatsReport,
+  SummaryStatsReport,
+} from "matrix-js-sdk/src/webrtc/stats/statsReport";
+
+export class ObjectFlattener {
+  public static flattenConnectionStatsReportObject(
+    statsReport: GroupCallStatsReport<ConnectionStatsReport>
+  ): Attributes {
+    const flatObject = {};
+    ObjectFlattener.flattenObjectRecursive(
+      statsReport.report,
+      flatObject,
+      "matrix.stats.conn.",
+      0
+    );
+    return flatObject;
+  }
+
+  public static flattenByteSentStatsReportObject(
+    statsReport: GroupCallStatsReport<ByteSentStatsReport>
+  ): Attributes {
+    const flatObject = {};
+    ObjectFlattener.flattenObjectRecursive(
+      statsReport.report,
+      flatObject,
+      "matrix.stats.bytesSent.",
+      0
+    );
+    return flatObject;
+  }
+
+  static flattenSummaryStatsReportObject(
+    statsReport: GroupCallStatsReport<SummaryStatsReport>
+  ) {
+    const flatObject = {};
+    ObjectFlattener.flattenObjectRecursive(
+      statsReport.report,
+      flatObject,
+      "matrix.stats.summary.",
+      0
+    );
+    return flatObject;
+  }
+
+  public static flattenObjectRecursive(
+    obj: Object,
+    flatObject: Attributes,
+    prefix: string,
+    depth: number
+  ): void {
+    if (depth > 10)
+      throw new Error(
+        "Depth limit exceeded: aborting VoipEvent recursion. Prefix is " +
+          prefix
+      );
+    let entries;
+    if (obj instanceof Map) {
+      entries = obj.entries();
+    } else {
+      entries = Object.entries(obj);
+    }
+    for (const [k, v] of entries) {
+      if (["string", "number", "boolean"].includes(typeof v) || v === null) {
+        let value;
+        value = v === null ? "null" : v;
+        value = typeof v === "number" && Number.isNaN(v) ? "NaN" : value;
+        flatObject[prefix + k] = value;
+      } else if (typeof v === "object") {
+        ObjectFlattener.flattenObjectRecursive(
+          v,
+          flatObject,
+          prefix + k + ".",
+          depth + 1
+        );
+      }
+    }
+  }
+}

src/otel/otel.ts

@@ -30,7 +30,7 @@ import { Anonymity } from "../analytics/PosthogAnalytics";
 import { Config } from "../config/Config";
 import { getSetting, settingsBus } from "../settings/useSetting";
 
-const SERVICE_NAME_BASE = "element-call";
+const SERVICE_NAME = "element-call";
 
 let sharedInstance: ElementCallOpenTelemetry;
 
@@ -48,7 +48,7 @@ export class ElementCallOpenTelemetry {
     return sharedInstance;
   }
 
-  constructor(collectorUrl: string) {
+  constructor(collectorUrl: string | undefined) {
     const otlpExporter = new OTLPTraceExporter({
       url: collectorUrl,
     });
@@ -58,7 +58,7 @@ export class ElementCallOpenTelemetry {
     // This is how we can make Jaeger show a reaonsable service in the dropdown on the left.
     const providerConfig = {
       resource: new Resource({
-        [SemanticResourceAttributes.SERVICE_NAME]: `${SERVICE_NAME_BASE}-unauthenticated`,
+        [SemanticResourceAttributes.SERVICE_NAME]: SERVICE_NAME,
       }),
     };
     this._provider = new WebTracerProvider(providerConfig);
@@ -88,12 +88,16 @@ export class ElementCallOpenTelemetry {
 }
 
 function recheckOTelEnabledStatus(optInAnalayticsEnabled: boolean): void {
-  if (optInAnalayticsEnabled && !sharedInstance) {
+  const shouldEnable =
+    optInAnalayticsEnabled &&
+    Boolean(Config.get().opentelemetry?.collector_url);
+
+  if (shouldEnable && !sharedInstance) {
     logger.info("Starting OpenTelemetry debug reporting");
     sharedInstance = new ElementCallOpenTelemetry(
       Config.get().opentelemetry?.collector_url
     );
-  } else if (!optInAnalayticsEnabled && sharedInstance) {
+  } else if (!shouldEnable && sharedInstance) {
     logger.info("Stopping OpenTelemetry debug reporting");
     sharedInstance = undefined;
   }