Outline
- After receiving the output by sending the video shot or loaded in iOS to the Plask API
- Converting Plask API output to animation of 3D model in Unity
Progress
3D Model set up
-
Before importing the model into the Unity Scene, it is necessary to align the local axis of the Plask API output with the local axis of the 3D model.
-
The rigging to align the axis of the 3D Model to be used is done according to the document below.
Import a model and build a scene on Unity
-
Import 3D model to apply Plask API output from Unity.
-
Example model
-
-
The script that is written according to this guide is added as a component of the 3D model.
-
Build your Unity Scene into the iOS platform and add it to your iOS project.
API output structure
-
The API output is a json file with the structure shown in the example below.
{
"result": [
{
"motionNumber": 0,
"trackData": [
{
"boneName": "hips",
"fps": 30,
"property": "position",
"transformKeys": [
{
"frame": 0,
"time": 0,
"value": [
0.06301826238632202,
0,
0.2913147509098053
]
}
]
}
]
}
],
"workingtime": 20.022384643554688
}-
Example
{
"result": [
{
"motionNumber": 0,
"trackData": [
{
"boneName": "hips",
"fps": 30,
"property": "position",
"transformKeys": [
{
"frame": 0,
"time": 0,
"value": [
0.06301826238632202,
0,
0.2913147509098053
]
}
]
}
]
}
],
"workingtime": 20.022384643554688
}
-
-
Motion capture data is included in “result”>”trackData”. It has position values (x, y, z) or rotation values (x, y, z, w) based on a skeleton with 24 bones. The bones are composed as follows.
Source bone structure
-
In the “result”>”trackData” array, hips have position and quaternion type values, and 23 bones excluding hips have only quaternion type values, so there are a total of 25 elements. Each element consists of boneName, fps, property and transformKeys.
"trackData": [
{
"boneName": "hips",
"fps": 30,
"property": "position",
"transformKeys": [
{
"frame": 0,
"time": 0,
"value": [
0.0630182 6238632202,
0,
0.2913147509098053
]
},
...
]
},
{
"boneName": "hips",
"fps": 30,
"property": "rotationQuaternion",
"transformKeys": [ ...
]
},
{
"boneName": "leftUpLeg",
"fps": 30,
"property": "rotationQuaternion",
"transformKeys": [ ...
]
},
...
{
"boneName": "rightHandIndex1",
"fps": 30,
"property": "rotationQuaternion",
"transformKeys": [ ...
]
},- boneName is the name of the bone with motion capture data.
- fps is the fps of the video to extract motion.
- property is the type of transform value.
- transformKeys is an array with time and value for each pose. Each pose has frame, time, and value information.
- frame is the frame order of the pose.
- time is the time to pose. The unit is seconds.
- value is the transform value of the pose. When the property is position, it has 3 values (x, y, z), and when it is rotationQuaternion, it has 4 values (x, y, z, w).
API output networking: iOS > Unity
-
Call the Unity screen built above
class AppDelegate: UIResponder, UIApplicationDelegate {
//method to call unity screen
func initAndShowUnity() -> Void {
if let framework = self.unityFrameworkLoad() {
self.unityFramework = framework
self.unityFramework?.setDataBundleId("com.unity3d.framework")
self.unityFramework?.runEmbedded(withArgc: CommandLine.argc,
argv: CommandLine.unsafeArgv,
appLaunchOpts: [:])
self.unityFramework?.showUnityWindow()
print("windows: \\(application?.windows)")
}
}
}
class VideoPreviewViewController: UIViewController, AVAudioPlayerDelegate {
private func pushSceneViewController(mocapJsonString: String?,
duration: Double) {
if let appDelegate = UIApplication.shared.delegate as? AppDelegate {
appDelegate.initAndShowUnity()
unityViewController = appDelegate.unityFramework?.appController()?.rootViewController
viewModel.send(event: VideoPreviewEvent.setUnityNotification)
viewModel.send(event: VideoPreviewEvent.handleUnityReady)
}
}
} -
After networking with Plask API is done on iOS, the response data is converted from json to String and delivered to Unity script. (unityFramework)
class VideoPreviewViewModel: NSObject {
private func handleUnityReady() {
guard let jsonString = self.jsonString else { return } //API output converted to String
unityManager.makeAnimationClip(jsonString: jsonString)
}
}
class UnityManager {
private var appDelegate: AppDelegate? {
return UIApplication.shared.delegate as? AppDelegate
}
func makeAnimationClip(jsonString: String?) {
appDelegate?.unityFramework?.sendMessageToGO(withName: "vangaurd_t_choonyung", //3d model
functionName: "MakeAnimationClip", //method
message: jsonString) //API output converted to String
}
}
Writing Unity script to apply API output
Result: Source code: Unity script
-
Defines bonesDictionary in the form of {<Target bone>:<Source bone>} for 24 source bones contained in API output and target bones of 3D model matching them
- Replace the keys in the dictionary with your 3D model’s bone.
public class MocapAnimation : MonoBehaviour {
Dictionary<string, string> bonesDictionary = new Dictionary<string, string>()
{
{ "mixamorig:Hips", "hips" },
{ "mixamorig:LeftUpLeg", "leftUpLeg" },
{ "mixamorig:RightUpLeg", "rightUpLeg" },
{ "mixamorig:Spine", "spine" },
{ "mixamorig:LeftLeg", "leftLeg" },
{ "mixamorig:RightLeg", "rightLeg" },
{ "mixamorig:Spine1", "spine1" },
{ "mixamorig:LeftFoot", "leftFoot" },
{ "mixamorig:RightFoot", "rightFoot" },
{ "mixamorig:Spine2", "spine2" },
{ "mixamorig:LeftToeBase", "leftToeBase" },
{ "mixamorig:RightToeBase", "rightToeBase" },
{ "mixamorig:Neck", "neck" },
{ "mixamorig:LeftShoulder", "leftShoulder" },
{ "mixamorig:RightShoulder", "rightShoulder" },
{ "mixamorig:Head", "head" },
{ "mixamorig:LeftArm", "leftArm" },
{ "mixamorig:RightArm", "rightArm" },
{ "mixamorig:RightForeArm", "rightForeArm" },
{ "mixamorig:LeftForeArm", "leftForeArm" },
{ "mixamorig:LeftHand", "leftHand" },
{ "mixamorig:RightHand", "rightHand" },
{ "mixamorig:LeftHandMiddle1", "leftHandIndex1" },
{ "mixamorig:RightHandMiddle1", "rightHandIndex1" }
};
...
}-
Example model
-
After deserializing JSON string, create a dictionary in the form of <string, trackData> and parse each data to search with ease
-
string: “<source bone name>.<transform type>”
- Source bone: Name representing 24 joints in API output
- Transform type: Properties such as position and rotationQuaternion
-
trackData : class with boneName, fps, property, transformKeys (custom defined)
- boneName: “<source bone>”
- fps: frames per second
- property: “<transform type>”
- transformKeys: data of frame (custom defined)
- frame: frame number
- times: Array of times with keyframes
- values: position or quaternion values that match times
-
<string, trackData> Example
// 1
"hips.position"
: trackData(boneName: "hips", fps: 30, property: "position", transformKeys: [(frame: 0, time: 0, values: [0.0, 1.0, 3.0]), ...])
// 2
"rightHandIndex1.rotationQuaternion"
: mocapData(boneName: "rightHandIndex1", fps: 30, property: "rotationQuaternion", transformKeys: [(frame: 3, time: 0.166, values: [0.0, 15.0, 30.0, 45.0]), ...)
[Serializable]
public class MocapResult {
public string id;
public MocapData[] result;
public float workingtime;
}
[Serializable]
public class MocapData {
public int motionNumber;
public TrackData[] trackData;
}
[Serializable]
public class TrackData {
public string boneName;
public int fps;
public string property;
public TransformKeys[] transformKeys;
}
[Serializable]
public class TransformKeys {
public int frame;
public float time;
public float[] value;
}
public class MocapAnimation : MonoBehaviour {
// ...
void MakeAnimationClip(string jsonString) {
MocapResult mocapResult = JsonUtility.FromJson<MocapResult>(jsonString);
Dictionary<String, TrackData> mocapDictionary = new Dictionary<string, TrackData>();
MocapData result = mocapResult.result[0];
for (int i = 0; i < result.trackData.Length; i++) {
string key = result.trackData[i].boneName + "." + result.trackData[i].property;
mocapDictionary.Add(key, result.trackData[i]);
}
if (transform.childCount > 0) {
ApplyAnimationFromTransform(transform, mocapDictionary);
}
}
// ...
} -
-
Following the {<target bone>:<source bone>} pair of bonesDictionary, trackData in the API output are passed to the function applied to the Animation component of the target bone.
- 3D model must have an Animation component in order to have animation.
- Check if the bone is in the boneDictionary by going through all the bones in the 3D model.
- If it exists in boneDictionary, it is passed to a function that applies trackData to the Animation component.
public class MocapAnimation : MonoBehaviour {
// ...
private void ApplyAnimationFromTransform(Transform parent, Dictionary<String, TrackData> trackDataDictionary) {
if (parent.childCount > 0) {
foreach (Transform child in parent) {
child.gameObject.AddComponent(typeof(Animation));
string childName = child.gameObject.name;
if (bonesDictionary.ContainsKey(childName) == true) {
string boneName = bonesDictionary[childName];
if (boneName == "hips") {
SetObjectPositionAndQuaternionAnimation(child, trackDataDictionary);
} else {
TrackData trackData = trackDataDictionary[boneName + ".rotationQuaternion"];
SetObjectQuaternionAnimation(child, trackData);
}
}
ApplyAnimationFromTransform(child, trackDataDictionary);
}
}
}
//...
} -
A function that creates and applies a component
- SetObjectPositionAndQuaternionAnimation is a function used when creating an animation clip of the target bone where the source bone corresponds to the hips.
- SetObjectQuaternionAnimation is a function used to create the animation component of the target bone corresponding to 23 joints where the source bone is not hips.
- Process
- Calls the Animation component of the 3D model created above.
- Keyframe arrays are created for each property of position(x, y, z) and quaternion (x, y, z, w), and the values of API output are sequentially inserted.
- Create AnimationCurve reflecting keyframe arrays
- Create AnimationClip reflecting AnimationCurve
- Keyword
- Keyframe Array: Array with the values of position and values sequentially
- AnimationCurve: Store a collection of Keyframes that can be evaluated over time to make AnimationClip
public class MocapAnimation : MonoBehaviour {
// ...
//Create and apply position and quaternion animation of hips bones
private void SetObjectPositionAndQuaternionAnimation(Transform transform, Dictionary<String, TrackData> trackDataDictionary) {
TrackData positionData = trackDataDictionary["hips.position"];
TrackData quaternionData = trackDataDictionary["hips.rotationQuaternion"];
Animation anim = transform.GetComponent<Animation>();
Keyframe[] positionKeysX = new Keyframe[positionData.transformKeys.Length];
Keyframe[] positionKeysY = new Keyframe[positionData.transformKeys.Length];
Keyframe[] positionKeysZ = new Keyframe[positionData.transformKeys.Length];
Keyframe[] quaternionKeysX = new Keyframe[quaternionData.transformKeys.Length];
Keyframe[] quaternionKeysY = new Keyframe[quaternionData.transformKeys.Length];
Keyframe[] quaternionKeysZ = new Keyframe[quaternionData.transformKeys.Length];
Keyframe[] quaternionKeysW = new Keyframe[quaternionData.transformKeys.Length];
for (int i = 0; i < positionData.transformKeys.Length; i++) {
TransformKeys positionTransformKey = positionData.transformKeys[i];
float positionTime = positionTransformKey.time;
positionKeysX[i] = new Keyframe(positionTime, positionTransformKey.value[0]);
positionKeysY[i] = new Keyframe(positionTime, positionTransformKey.value[1]);
positionKeysZ[i] = new Keyframe(positionTime, positionTransformKey.value[2]);
TransformKeys quaternionTransformKey = quaternionData.transformKeys[i];
float quaternionTime = quaternionTransformKey.time;
quaternionKeysX[i] = new Keyframe(quaternionTime, quaternionTransformKey.value[0]);
quaternionKeysY[i] = new Keyframe(quaternionTime, -quaternionTransformKey.value[1]);
quaternionKeysZ[i] = new Keyframe(quaternionTime, -quaternionTransformKey.value[2]);
quaternionKeysW[i] = new Keyframe(quaternionTime, quaternionTransformKey.value[3]);
}
var positionCurveX = new AnimationCurve(positionKeysX);
var positionCurveY = new AnimationCurve(positionKeysY);
var positionCurveZ = new AnimationCurve(positionKeysZ);
var quaternionCurveX = new AnimationCurve(quaternionKeysX);
var quaternionCurveY = new AnimationCurve(quaternionKeysY);
var quaternionCurveZ = new AnimationCurve(quaternionKeysZ);
var quaternionCurveW = new AnimationCurve(quaternionKeysW);
AnimationClip animationClip = new AnimationClip();
animationClip.legacy = true;
animationClip.SetCurve("", typeof(Transform), "localPosition.x", positionCurveX);
animationClip.SetCurve("", typeof(Transform), "localPosition.y", positionCurveY);
animationClip.SetCurve("", typeof(Transform), "localPosition.z", positionCurveZ);
animationClip.SetCurve("", typeof(Transform), "localRotation.x", quaternionCurveX);
animationClip.SetCurve("", typeof(Transform), "localRotation.y", quaternionCurveY);
animationClip.SetCurve("", typeof(Transform), "localRotation.z", quaternionCurveZ);
animationClip.SetCurve("", typeof(Transform), "localRotation.w", quaternionCurveW);
anim.AddClip(animationClip, "mocapAnimation");
anim.Play("mocapAnimation");
}
//Create and apply bones quaternion animation except for hips
private void SetObjectQuaternionAnimation(Transform child, TrackData trackData) {
Animation anim = child.GetComponent<Animation>();
int frameLength = trackData.transformKeys.Length;
Keyframe[] keysX = new Keyframe[frameLength];
Keyframe[] keysY = new Keyframe[frameLength];
Keyframe[] keysZ = new Keyframe[frameLength];
Keyframe[] keysW = new Keyframe[frameLength];
for (int i = 0; i < frameLength; i++) {
TransformKeys transformKey = trackData.transformKeys[i];
float time = transformKey.time;
keysX[i] = new Keyframe(time, transformKey.value[0]);
keysY[i] = new Keyframe(time, -transformKey.value[1]);
keysZ[i] = new Keyframe(time, -transformKey.value[2]);
keysW[i] = new Keyframe(time, transformKey.value[3]);
}
var curveX = new AnimationCurve(keysX);
var curveY = new AnimationCurve(keysY);
var curveZ = new AnimationCurve(keysZ);
var curveW = new AnimationCurve(keysW);
AnimationClip animationClip = new AnimationClip();
animationClip.legacy = true;
animationClip.SetCurve("", typeof(Transform), "localRotation.x", curveX);
animationClip.SetCurve("", typeof(Transform), "localRotation.y", curveY);
animationClip.SetCurve("", typeof(Transform), "localRotation.z", curveZ);
animationClip.SetCurve("", typeof(Transform), "localRotation.w", curveW);
anim.AddClip(animationClip, "mocapAnimation");
anim.Play("mocapAnimation");
}
}
Source code: Unity script
MocapAnimation.cs
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using UnityEngine;
using UnityEditor;
[Serializable]
public class MocapResult {
public string id;
public MocapData[] result;
public float workingtime;
}
[Serializable]
public class MocapData {
public int motionNumber;
public TrackData[] trackData;
}
[Serializable]
public class TrackData {
public string boneName;
public int fps;
public string property;
public TransformKeys[] transformKeys;
}
[Serializable]
public class TransformKeys {
public int frame;
public float time;
public float[] value;
}
public class MocapAnimation : MonoBehaviour {
Dictionary<string, string> bonesDictionary = new Dictionary<string, string>()
{
{ "mixamorig:Hips", "hips" },
{ "mixamorig:LeftUpLeg", "leftUpLeg" },
{ "mixamorig:RightUpLeg", "rightUpLeg" },
{ "mixamorig:Spine", "spine" },
{ "mixamorig:LeftLeg", "leftLeg" },
{ "mixamorig:RightLeg", "rightLeg" },
{ "mixamorig:Spine1", "spine1" },
{ "mixamorig:LeftFoot", "leftFoot" },
{ "mixamorig:RightFoot", "rightFoot" },
{ "mixamorig:Spine2", "spine2" },
{ "mixamorig:LeftToeBase", "leftToeBase" },
{ "mixamorig:RightToeBase", "rightToeBase" },
{ "mixamorig:Neck", "neck" },
{ "mixamorig:LeftShoulder", "leftShoulder" },
{ "mixamorig:RightShoulder", "rightShoulder" },
{ "mixamorig:Head", "head" },
{ "mixamorig:LeftArm", "leftArm" },
{ "mixamorig:RightArm", "rightArm" },
{ "mixamorig:RightForeArm", "rightForeArm" },
{ "mixamorig:LeftForeArm", "leftForeArm" },
{ "mixamorig:LeftHand", "leftHand" },
{ "mixamorig:RightHand", "rightHand" },
{ "mixamorig:LeftHandMiddle1", "leftHandIndex1" },
{ "mixamorig:RightHandMiddle1", "rightHandIndex1" }
};
// iOS에서 받아온 jsonString -> deserialize
void MakeAnimationClip(string jsonString) {
MocapResult mocapResult = JsonUtility.FromJson<MocapResult>(jsonString);
Dictionary<String, TrackData> mocapDictionary = new Dictionary<string, TrackData>();
MocapData result = mocapResult.result[0];
for (int i = 0; i < result.trackData.Length; i++) {
string key = result.trackData[i].boneName + "." + result.trackData[i].property;
mocapDictionary.Add(key, result.trackData[i]);
}
if (transform.childCount > 0) {
ApplyAnimationFromTransform(transform, mocapDictionary);
}
}
// bones 비교
private void ApplyAnimationFromTransform(Transform parent, Dictionary<String, TrackData> trackDataDictionary) {
if (parent.childCount > 0) {
foreach (Transform child in parent) {
child.gameObject.AddComponent(typeof(Animation));
string childName = child.gameObject.name;
if (bonesDictionary.ContainsKey(childName) == true) {
string boneName = bonesDictionary[childName];
if (boneName == "hips") {
SetObjectPositionAndQuaternionAnimation(child, trackDataDictionary);
} else {
TrackData trackData = trackDataDictionary[boneName + ".rotationQuaternion"];
SetObjectQuaternionAnimation(child, trackData);
}
}
ApplyAnimationFromTransform(child, trackDataDictionary);
}
}
}
// hips의 position, rotation Animation 생성 및 적용
private void SetObjectPositionAndQuaternionAnimation(Transform transform, Dictionary<String, TrackData> trackDataDictionary) {
TrackData positionData = trackDataDictionary["hips.position"];
TrackData quaternionData = trackDataDictionary["hips.rotationQuaternion"];
Animation anim = transform.GetComponent<Animation>();
Keyframe[] positionKeysX = new Keyframe[positionData.transformKeys.Length];
Keyframe[] positionKeysY = new Keyframe[positionData.transformKeys.Length];
Keyframe[] positionKeysZ = new Keyframe[positionData.transformKeys.Length];
Keyframe[] quaternionKeysX = new Keyframe[quaternionData.transformKeys.Length];
Keyframe[] quaternionKeysY = new Keyframe[quaternionData.transformKeys.Length];
Keyframe[] quaternionKeysZ = new Keyframe[quaternionData.transformKeys.Length];
Keyframe[] quaternionKeysW = new Keyframe[quaternionData.transformKeys.Length];
for (int i = 0; i < positionData.transformKeys.Length; i++) {
TransformKeys positionTransformKey = positionData.transformKeys[i];
float positionTime = positionTransformKey.time;
positionKeysX[i] = new Keyframe(positionTime, positionTransformKey.value[0]);
positionKeysY[i] = new Keyframe(positionTime, positionTransformKey.value[1]);
positionKeysZ[i] = new Keyframe(positionTime, positionTransformKey.value[2]);
TransformKeys quaternionTransformKey = quaternionData.transformKeys[i];
float quaternionTime = quaternionTransformKey.time;
quaternionKeysX[i] = new Keyframe(quaternionTime, quaternionTransformKey.value[0]);
quaternionKeysY[i] = new Keyframe(quaternionTime, -quaternionTransformKey.value[1]);
quaternionKeysZ[i] = new Keyframe(quaternionTime, -quaternionTransformKey.value[2]);
quaternionKeysW[i] = new Keyframe(quaternionTime, quaternionTransformKey.value[3]);
}
var positionCurveX = new AnimationCurve(positionKeysX);
var positionCurveY = new AnimationCurve(positionKeysY);
var positionCurveZ = new AnimationCurve(positionKeysZ);
var quaternionCurveX = new AnimationCurve(quaternionKeysX);
var quaternionCurveY = new AnimationCurve(quaternionKeysY);
var quaternionCurveZ = new AnimationCurve(quaternionKeysZ);
var quaternionCurveW = new AnimationCurve(quaternionKeysW);
AnimationClip animationClip = new AnimationClip();
animationClip.legacy = true;
animationClip.SetCurve("", typeof(Transform), "localPosition.x", positionCurveX);
animationClip.SetCurve("", typeof(Transform), "localPosition.y", positionCurveY);
animationClip.SetCurve("", typeof(Transform), "localPosition.z", positionCurveZ);
animationClip.SetCurve("", typeof(Transform), "localRotation.x", quaternionCurveX);
animationClip.SetCurve("", typeof(Transform), "localRotation.y", quaternionCurveY);
animationClip.SetCurve("", typeof(Transform), "localRotation.z", quaternionCurveZ);
animationClip.SetCurve("", typeof(Transform), "localRotation.w", quaternionCurveW);
anim.AddClip(animationClip, "mocapAnimation");
anim.Play("mocapAnimation");
}
// hips 외의 bones rotation Animation 생성 및 적용
private void SetObjectQuaternionAnimation(Transform child, TrackData trackData) {
Animation anim = child.GetComponent<Animation>();
int frameLength = trackData.transformKeys.Length;
Keyframe[] keysX = new Keyframe[frameLength];
Keyframe[] keysY = new Keyframe[frameLength];
Keyframe[] keysZ = new Keyframe[frameLength];
Keyframe[] keysW = new Keyframe[frameLength];
for (int i = 0; i < frameLength; i++) {
TransformKeys transformKey = trackData.transformKeys[i];
float time = transformKey.time;
keysX[i] = new Keyframe(time, transformKey.value[0]);
keysY[i] = new Keyframe(time, -transformKey.value[1]);
keysZ[i] = new Keyframe(time, -transformKey.value[2]);
keysW[i] = new Keyframe(time, transformKey.value[3]);
}
var curveX = new AnimationCurve(keysX);
var curveY = new AnimationCurve(keysY);
var curveZ = new AnimationCurve(keysZ);
var curveW = new AnimationCurve(keysW);
AnimationClip animationClip = new AnimationClip();
animationClip.legacy = true;
animationClip.SetCurve("", typeof(Transform), "localRotation.x", curveX);
animationClip.SetCurve("", typeof(Transform), "localRotation.y", curveY);
animationClip.SetCurve("", typeof(Transform), "localRotation.z", curveZ);
animationClip.SetCurve("", typeof(Transform), "localRotation.w", curveW);
anim.AddClip(animationClip, "mocapAnimation");
anim.Play("mocapAnimation");
}
}