What joints do different devices plan to support?

[Manishearth]

I suspect various devices with articulated hand tracking will support different sets of joints. We should figure out the superset.

@thetuvix @Artyom17 Can y'all list what your devices plan to support? I also have a list from Microsoft's draft openxr extension, but I'm not certain if this is something I can make public

[Artyom17]

Here is an excerpt from the Oculus Mobile SDK with Hands Tracking support.

typedef enum ovrHandFingers_
{
	ovrHandFinger_Thumb		= 0,
	ovrHandFinger_Index		= 1,
	ovrHandFinger_Middle	= 2,
	ovrHandFinger_Ring		= 3,
	ovrHandFinger_Pinky		= 4,
	ovrHandFinger_Max,
	ovrHandFinger_EnumSize = 0x7fffffff
} ovrHandFingers;

typedef enum ovrHandPinchStrength_
{
	ovrHandPinchStrength_Index	= 0,	// hand is in the index finger pinch state
	ovrHandPinchStrength_Middle	= 1,	// hand is in the middle finger pinch state
	ovrHandPinchStrength_Ring	= 2,	// hand is in the ring finger pinch state
	ovrHandPinchStrength_Pinky	= 3,	// hand is in the pinky finger pinch state
	ovrHandPinchStrength_Max 	= 4,
	ovrHandPinchStrength_EnumSize 	= 0x7fffffff
} ovrHandPinchStrength;

typedef int16_t ovrVertexIndex;

typedef enum ovrHandBone_
{
	ovrHandBone_Invalid						= -1,
	ovrHandBone_WristRoot 					= 0,	// root frame of the hand, where the wrist is located
	ovrHandBone_ForearmStub					= 1,	// frame for user's forearm
	ovrHandBone_Thumb0						= 2,	// thumb trapezium bone
	ovrHandBone_Thumb1						= 3,	// thumb metacarpal bone
	ovrHandBone_Thumb2						= 4,	// thumb proximal phalange bone
	ovrHandBone_Thumb3						= 5,	// thumb distal phalange bone
	ovrHandBone_Index1						= 6,	// index proximal phalange bone
	ovrHandBone_Index2						= 7,	// index intermediate phalange bone
	ovrHandBone_Index3						= 8,	// index distal phalange bone
	ovrHandBone_Middle1						= 9,	// middle proximal phalange bone
	ovrHandBone_Middle2						= 10,	// middle intermediate phalange bone
	ovrHandBone_Middle3						= 11,	// middle distal phalange bone
	ovrHandBone_Ring1						= 12,	// ring proximal phalange bone
	ovrHandBone_Ring2						= 13,	// ring intermediate phalange bone
	ovrHandBone_Ring3						= 14,	// ring distal phalange bone
	ovrHandBone_Pinky0						= 15,	// pinky metacarpal bone
	ovrHandBone_Pinky1						= 16,	// pinky proximal phalange bone
	ovrHandBone_Pinky2						= 17,	// pinky intermediate phalange bone
	ovrHandBone_Pinky3						= 18,	// pinky distal phalange bone
	ovrHandBone_MaxSkinnable				= 19,

	// Bone tips are position only. They are not used for skinning but useful for hit-testing.
	// NOTE: ovrHandBone_ThumbTip == ovrHandBone_MaxSkinnable since the extended tips need to be contiguous
	ovrHandBone_ThumbTip					= ovrHandBone_MaxSkinnable + 0,	// tip of the thumb
	ovrHandBone_IndexTip					= ovrHandBone_MaxSkinnable + 1,	// tip of the index finger
	ovrHandBone_MiddleTip					= ovrHandBone_MaxSkinnable + 2,	// tip of the middle finger
	ovrHandBone_RingTip						= ovrHandBone_MaxSkinnable + 3,	// tip of the ring finger
	ovrHandBone_PinkyTip					= ovrHandBone_MaxSkinnable + 4,	// tip of the pinky
	ovrHandBone_Max 						= ovrHandBone_MaxSkinnable + 5,
	ovrHandBone_EnumSize 					= 0x7fff
} ovrHandBone;
typedef int16_t ovrHandBoneIndex;

// ovrBoneCapsule
//    _---_
//  -"     "-
// /         \
// |----A----|
// |    |    |
// |    |    |
// |    |-r->|
// |    |    |
// |    |    |
// |----B----|
// \         /
//  -.     .-
//    '---'
typedef struct ovrBoneCapsule_
{
	// Index of the bone this capsule is on.
	ovrHandBoneIndex 	BoneIndex;
	// Points at either end of the cylinder inscribed in the capsule. Also the center points for
	// spheres at either end of the capsule. Points A and B in the diagram above.
	ovrVector3f			Points[2];
	// The radius of the capsule cylinder and of the half-sphere caps on the ends of the capsule.
	float				Radius;
} ovrBoneCapsule;

typedef enum ovrHandConstants_
{
	ovrHand_MaxVertices				= 3000,
	ovrHand_MaxIndices				= ovrHand_MaxVertices * 6,
	ovrHand_MaxFingers				= ovrHandFinger_Max,
	ovrHand_MaxPinchStrengths		= ovrHandPinchStrength_Max,
	ovrHand_MaxSkinnableBones		= ovrHandBone_MaxSkinnable,
	ovrHand_MaxBones				= ovrHandBone_Max,
	ovrHand_MaxCapsules				= 19,
	ovrHand_EnumSize 				= 0x7fffffff
} ovrHandConstants;

// Pass this structure to vrapi_GetHandPose() to get the pose of the hand at a particular time.
typedef struct ovrHandPose_
{
	ovrHandPoseHeader			Header;

	// Status of tracking for this pose. This is not a bit field, but an exclusive state.
	ovrHandTrackingStatus		Status;

	// Root pose of the hand in world space. Not to be confused with the root bone's transform.
	// The root bone can still be offset from this by the skeleton's rest pose.
	ovrPosef					RootPose;

	// Current rotation of each bone.
	ovrQuatf					BoneRotations[ovrHandBone_Max];

	// Time stamp for the pose that was requested in global system time.
	double						RequestedTimeStamp;

	// Time stamp of the captured sample that the pose was extrapolated from.
	double						SampleTimeStamp;

	// Tracking confidence.
	// This is the amount of confidence that the system has that the entire hand pose is correct.
	ovrConfidence				HandConfidence;

	// Scale of the hand relative to the original hand model. This value may change at any time
	// based on the size of the hand being tracked. The default is 1.0. 
	float						HandScale;

	// Per-finger tracking confidence.
	// This is the amount of confidence the system has that the individual finger poses are correct.
	ovrConfidence				FingerConfidences[ovrHandFinger_Max];
} ovrHandPose;

OVR_VRAPI_EXPORT ovrResult vrapi_GetHandPose(	ovrMobile * ovr, const ovrDeviceID deviceID,
												const double absTimeInSeconds,
												ovrHandPoseHeader * header );

//-----------------------------------------------------------------
// Hand skeleton

// Header for all mesh structures.
typedef struct ovrHandSkeletonHeader_
{
	// The version number of the skeleton structure.
	ovrHandVersion		Version;
} ovrHandSkeletonHeader;

typedef struct ovrHandSkeleton_V1_
{
	// Version of the mesh structure.
	ovrHandSkeletonHeader	Header;

	// The number of bones in this skeleton.
	uint32_t				NumBones;

	// The number of capsules on this skeleton.
	uint32_t				NumCapsules;

	// reserved for future use
	uint32_t				Reserved[5];

	// An array of count NumBones transforms for each bone in local (parent) space.
	ovrPosef 				BonePoses[ovrHand_MaxBones];

	// An array of count NumBones indicating the parent bone index for each bone.
	ovrHandBoneIndex		BoneParentIndices[ovrHand_MaxBones];

	// An array of count NumCapsules ovrHandCapsules. Note that the number of capsules
	// is not necessarily the same as the number of bones.
	ovrBoneCapsule  		Capsules[ovrHand_MaxCapsules];
} ovrHandSkeleton;

OVR_VRAPI_EXPORT ovrResult vrapi_GetHandSkeleton( ovrMobile * ovr, const ovrHandedness handedness, ovrHandSkeletonHeader * header );


//-----------------------------------------------------------------
// Hand mesh

// Header for all mesh structures.
typedef struct ovrHandMeshHeader_
{
	// The version number of the mesh structure.
	ovrHandVersion		Version;
} ovrHandMeshHeader;

typedef struct ovrHandMesh_V1_
{
	// All mesh structures will start with this header and the version.
	ovrHandMeshHeader		Header;

	// Number of unique vertices in the mesh.
	uint32_t				NumVertices;
	// Number of unique indices in the mesh.
	uint32_t				NumIndices;

	// Reserved for future use
	uint32_t				Reserved[13];

	// An array of count NumVertices positions for each vertex.
	ovrVector3f				VertexPositions[ovrHand_MaxVertices];
	// An array of count NumIndices of vertex indices specifying triangles that make up the mesh.
	ovrVertexIndex			Indices[ovrHand_MaxIndices];
	// An array of count NumVertices of normals for each vertex.
	ovrVector3f				VertexNormals[ovrHand_MaxVertices];
	// An array of count NumVertices of texture coordinates for each vertex.
	ovrVector2f				VertexUV0[ovrHand_MaxVertices];
	// An array of count NumVertices of blend indices for each of the bones that each vertex is weighted to.
	// Always valid. An index of < 0 means no blend weight.
	ovrVector4s				BlendIndices[ovrHand_MaxVertices];
	// An array of count NumVertices of weights for each of the bones affecting each vertex.
	ovrVector4f				BlendWeights[ovrHand_MaxVertices];
} ovrHandMesh;

OVR_VRAPI_EXPORT ovrResult vrapi_GetHandMesh( ovrMobile * ovr, const ovrHandedness handedness, ovrHandMeshHeader * header );

[Manishearth]

Going through the enums, both OpenXR and Oculus support

• a wrist joint
• metacarpal + proximal, distal, tip phalange for thumb
• metacarpal + proximal, intermediate, distal, tip phalange for non-thumbs

Additionally, Oculus supports the trapezium carpal for the thumb, as well as a "forearm stub" joint. OpenXR supports a palm "joint", which is situated on the middle metacarpal.

[Manishearth]

Actually, Oculus is missing metacarpals for index, middle, and ring fingers

[Manishearth]

@Artyom17 do you have a diagram explaining where exactly the poses for these are? And also how their orientation is calculated. Was trying to figure this out for https://github.com/immersive-web/webxr-hands-input/issues/9 but wasn't sure.

Text documentation would be good as well.

[Artyom17]

Let me ask

[Manishearth]

We will probably need to expose some notion of parent joint given that not all devices support not all joints and while users can figure it out based on the gaps, it's easier to just expose it.

[Artyom17]

There is some info here: https://developer.oculus.com/documentation/quest/latest/concepts/mobile-vrapi-input-api/ (still asking for more)

[Manishearth]

Hmm that link isn't working for me

[Artyom17]

Sorry, the proper link: https://developer.oculus.com/documentation/quest/latest/concepts/mobile-vrapi-input-api/

[Artyom17]

Posting the diagram here as well: The joints are oriented such that +X points down the bone on the left hand and -X points down the bone on the right hand.

Actually, Oculus is missing metacarpals for index, middle, and ring fingers.

There's negligible rotation in those metacarapals, so we don't bother to track or report them. We don't actually track the pinky metacarpal current, either (rotation is always reported as identity), but we've put it in there for forward compatibility.

Oculus supports the trapezium carpal for the thumb, as well as a "forearm stub" joint. OpenXR supports a palm "joint", which is situated on the middle metacarpal.

The forearm stub joint is also built in as a forward compatibility thing in case we want to counter-rotate the wrist if we add separate wrist tracking. Currently always identity.

[Artyom17]

One more diagram with names of those bones

[Manishearth]

So, interestingly, the naming oculus uses means that rotations for the pseudocarpals are missing. The pseudocarpals being the "bones" between the wrist and the metacarpals; they don't actually exist as distinct bones, but they're drawn in your diagram. You can derive their orientations by subtracting the wrist from the metacarpals or proximal phalanges; the only thing that really matters is the x direction, but it feels weird to not have them exposed. On the other hand, naming them is tricky (see https://github.com/immersive-web/webxr-hands-input/issues/9 )

[Manishearth]

Alright, so thinking more of this, we have the following "bones" overall:

• proximal, middle, distal phalanges (no middle on thumb). we need position and orientation
• metacarpals. we need position and orientation
• tip bones: position matters, orientation does not
• "pseudocarpal" bones connecting the bottom of the metacarpal to the wrist. The position is the same as the wrist for these, as are the orientation skin-normal values, but the orientation bone-direction value is different. This can, however, be calculated by subtracting the wrist position from the metacarpal

I think it makes sense to expose the phalanges, metacarpals, and tip, with each position located at the bottom of the bone (closer to the wrist). The pseudocarpal bones are displayed in many diagrams, but only their x value is necessary, and since you can calculate it it's not a big deal. This is true for orientations in general -- you only really need the skin normal -- but it's convenient to expose whatever we can if it's not too much work.

[Manishearth]

This means we won't expose:

• oculus' trapezium bone
• pseudocarpals if openxr decides to go with that route
• the forearm stub
• openxr's palm joint

implementors can choose to expose only some of the joints, and they may support more joints than what we expose; both systems work fine.

We do not have the problem oculus has with bone backwards compatibility because I'm hoping to not expose bones as a hierarchical concept the way oculus does (with each pose being relative to a "parent"), instead just being a list of joints each with an XRSpace, and you can choose to relate them with each other or with a fixed reference space at your discretion. So in the future we may add more bones if we wish.

[Manishearth]

Here's magic leap: https://developer.magicleap.com/learn/guides/lumin-sdk-handtracking