Envelop For Live and Unity 3D

After spending about 6 months learning the graphics stack and trying to make an enhanced editor in Three.js, I started running into a lot of problems that were already solved with tools like Flash. Three is great for doing 3D things on the web but the minute you need something more performant, such as recording video, the task becomes much more complicated due to the single threaded nature of JavaScript. After seeing many of the modern VR/AR creations on Twitter, the switch to Unity was the obvious next step.

Unity just released support for ambisonic audio and a new timeline experience that dramatically simplifies the creation of animations for many different platforms. These are the 2 biggest features that I thought were holding the platform back for creators.

Project Setup

Prerequisites:

Unity: Beginner
Ableton: Intermediate
OS Used for code examples: OSX

Software:

  • Ableton Live 9
  • Max For Live
  • Max 7
  • Unity3D
  • Visual Studio

Hardware:

  • Headphones
Repo: Github

To get started, we are going to port over the WebGL Envelop model that we made earlier this year from the Envelop LED control software, EnvelopLX.

Setting Up Envelop Model in Unity

Create a new Unity project and name it whatever you like. Since we already ported the WebGL model, I’ll be following along with that code. First let’s create the class for the Midway venue dimensions to give us a visual model for one of the decoders.

Assets > Create > C# Script, Midway.cs
using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class Midway : Venue
{
    private static float INCHES = 1.0f;
    private static float FEET = 12.0f * INCHES;
    private static float WIDTH = 20.0f * FEET + 10.25f * INCHES;
    private static float DEPTH = 41.0f * FEET + 6.0f * INCHES;

    private static float INNER_OFFSET_X = WIDTH / 2.0f - 1.0f * FEET - 8.75f * INCHES;
    private static float OUTER_OFFSET_X = WIDTH / 2.0f - 5.0f * FEET - 1.75f * INCHES;
    private static float INNER_OFFSET_Z = -DEPTH / 2.0f + 15.0f * FEET + 10.75f * INCHES;
    private static float OUTER_OFFSET_Z = -DEPTH / 2.0f + 7.0f * FEET + 8.0f * INCHES;

    private static float SUB_OFFSET_X = 36.0f * INCHES;
    private static float SUB_OFFSET_Z = 20.0f * INCHES;

    public ArrayList COLUMN_POSITIONS;
    public ArrayList SUB_POSITIONS;

    //TODO: Calculate values
    public static float xRange = 228.24f;
    public static float yRange = 141.73f;
    public static float zRange = 329.972f;
    public static float cx = 0f;
    public static float cy = 70.1f;
    public static float cz = 0f;

    // Use this for initialization  
    public void Create()
    {
        COLUMN_POSITIONS = new ArrayList();
        COLUMN_POSITIONS.Add(new Vector3(-OUTER_OFFSET_X, -OUTER_OFFSET_Z, 101));
        COLUMN_POSITIONS.Add(new Vector3(-INNER_OFFSET_X, -INNER_OFFSET_Z, 102));
        COLUMN_POSITIONS.Add(new Vector3(-INNER_OFFSET_X, INNER_OFFSET_Z, 103));
        COLUMN_POSITIONS.Add(new Vector3(-OUTER_OFFSET_X, OUTER_OFFSET_Z, 104));
        COLUMN_POSITIONS.Add(new Vector3(OUTER_OFFSET_X, OUTER_OFFSET_Z, 105));
        COLUMN_POSITIONS.Add(new Vector3(INNER_OFFSET_X, INNER_OFFSET_Z, 106));
        COLUMN_POSITIONS.Add(new Vector3(INNER_OFFSET_X, -INNER_OFFSET_Z, 107));
        COLUMN_POSITIONS.Add(new Vector3(OUTER_OFFSET_X, -OUTER_OFFSET_Z, 108));

        SUB_POSITIONS = new ArrayList();
        SUB_POSITIONS.Add((Vector3) COLUMN_POSITIONS[0] + new Vector3(-SUB_OFFSET_X, -SUB_OFFSET_Z, 0));
        SUB_POSITIONS.Add((Vector3) COLUMN_POSITIONS[3] + new Vector3(-SUB_OFFSET_X, SUB_OFFSET_Z, 0));
        SUB_POSITIONS.Add((Vector3) COLUMN_POSITIONS[4] + new Vector3(SUB_OFFSET_X, SUB_OFFSET_Z, 0));
        SUB_POSITIONS.Add((Vector3) COLUMN_POSITIONS[7] + new Vector3(SUB_OFFSET_X, -SUB_OFFSET_Z, 0));
    }
}

Now that we have our venue data setup, we need to translate it into Unity GameObjects. Following along with the Envelop class created here in JavaScript. For a more in depth overview of the creation of the JavaScript, see the first tutorial on Spatial Audio and Web VR.

Assets > Create > C# Script, Envelop.cs
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using System;

public class Envelop : MonoBehaviour {

    //Constants
    private static float INCHES = 1.0f;
    private static float FEET = 12.0f * INCHES;
    private static float SPEAKER_ANGLE = 22.0f;
    private static float RADIUS = 20.0f * INCHES;
    private static float HEIGHT = 12.0f * FEET;
    private static float NUM_INPUTS = 2;

    //Model Objects
    GameObject EnvelopModel;
    Midway EnvelopVenue;
    ArrayList columns;

    // Use this for initialization
    void Start () {
        EnvelopModel = new GameObject("Envelop");
        EnvelopVenue = new Midway();
        EnvelopVenue.Create();

        ColumnModels();
        ChannelModels();
        SubModels();
        InputModels();
    }

    // Update is called once per frame
    void Update () {

    }

    void InputModels() {
        for (var x = 0; x < NUM_INPUTS; x++) {
            GameObject sphere = GameObject.CreatePrimitive(PrimitiveType.Sphere);
            sphere.transform.localScale += new Vector3(RADIUS, RADIUS, RADIUS);
            sphere.transform.parent = EnvelopModel.transform;
        }
    }

    void ColumnModels() {
        columns = new ArrayList();
        foreach(Vector3 position in EnvelopVenue.COLUMN_POSITIONS) {
            GameObject cylinder;
            cylinder = GameObject.CreatePrimitive(PrimitiveType.Cylinder);
            cylinder.transform.localScale += new Vector3(RADIUS, HEIGHT / 2, RADIUS);
            cylinder.transform.position = new Vector3(position.x, HEIGHT / 2, position.y);

            float theta = (float) Math.Atan2(cylinder.transform.position.x, cylinder.transform.position.y) - (float)Math.PI / 2;
            cylinder.transform.Rotate(0, theta * (180 / (float)Math.PI), 0, Space.Self);
            cylinder.transform.parent = EnvelopModel.transform;
            columns.Add(cylinder);
        }
    }

    void ChannelModels() {
        foreach(GameObject column in columns) {
            //RADIANS
            //float columnTheta = (float)Math.Atan2(column.transform.position.y, column.transform.position.x) - (float)Math.PI / 2;

            //CONVERT RADIANS to DEGRESS
            //float columnDegrees = columnTheta * (180 / (float)Math.PI);

            GameObject channelBox1 = GameObject.CreatePrimitive(PrimitiveType.Cube);
            channelBox1.transform.parent = EnvelopModel.transform;
            channelBox1.transform.localScale = new Vector3(21 * INCHES, 16 * INCHES, 15 * INCHES);
            channelBox1.transform.position = new Vector3(column.transform.position.x, 1 * FEET, column.transform.position.z);
            channelBox1.transform.LookAt(Vector3.zero);
            channelBox1.transform.Rotate(-SPEAKER_ANGLE, 0, 0, Space.Self);
            //channelBox1.transform.Rotate(-SPEAKER_ANGLE, -columnDegrees, 0, Space.Self);

            GameObject channelBox2 = GameObject.CreatePrimitive(PrimitiveType.Cube);
            channelBox2.transform.parent = EnvelopModel.transform;
            channelBox2.transform.localScale = new Vector3(21 * INCHES, 16 * INCHES, 15 * INCHES);
            channelBox2.transform.position = new Vector3(column.transform.position.x, 6 * FEET, column.transform.position.z);
            channelBox2.transform.LookAt(new Vector3(0, HEIGHT / 2, 0));


            GameObject channelBox3 = GameObject.CreatePrimitive(PrimitiveType.Cube);
            channelBox3.transform.parent = EnvelopModel.transform;
            channelBox3.transform.localScale = new Vector3(21 * INCHES, 16 * INCHES, 15 * INCHES);
            channelBox3.transform.position = new Vector3(column.transform.position.x, 11 * FEET, column.transform.position.z);
            channelBox3.transform.LookAt(Vector3.zero);
            channelBox3.transform.Rotate(-SPEAKER_ANGLE, 0, 0, Space.Self);
            //channelBox3.transform.Rotate(-SPEAKER_ANGLE, -columnDegrees, 0, Space.Self);
        }
    }

    void SubModels() {
        foreach(Vector3 subPosition in EnvelopVenue.SUB_POSITIONS) {
            GameObject subBox = GameObject.CreatePrimitive(PrimitiveType.Cube);
            subBox.transform.parent = EnvelopModel.transform;
            subBox.transform.localScale = new Vector3(29 * INCHES, 20 * INCHES, 29 * INCHES);
            subBox.transform.position = new Vector3(subPosition.x, 10 * INCHES, subPosition.y);
            subBox.transform.LookAt(Vector3.zero);
        }
    }
}

After creating these two classes, hitting play in Unity should give you a replica of the Envelop at the Midway model.

E4L Unity Model

Connecting Unity to E4L Data via OSC

Now that we have our model in place we can connect the OSC data coming from E4L to our Unity model. For this we are going to use the UnityOSC package by Jorge Garcia. Follow the readme instructions for porting the OSC package into your project. My project directory now looks like this after adding the OSC package.

Project Files View

Once you have the correct files in place, we need to create a server to listen for the OSC messages from E4L. We also need to add the Max UDP patch to the E4L setup in order to parse and send the positional data to our UDP server.

Assets > Create > C# Script, oscControl.cs
using UnityEngine;
using System;
using System.Net;
using System.Collections;
using System.Collections.Generic;
using System.Text;
using UnityOSC;

public class oscControl : MonoBehaviour
{
    private OSCServer myServer;

    public string outIP = "127.0.0.1";
    public int outPort = 9999;
    public int inPort = 57121;

    // Buffer size of the application (stores 100 messages from different servers)
    public int bufferSize = 100;

    // Script initialization
    void Start()
    {
        // init OSC
        OSCHandler.Instance.Init();

        // Initialize OSC clients (transmitters)
        OSCHandler.Instance.CreateClient("myClient", IPAddress.Parse(outIP), outPort);

        // Initialize OSC servers (listeners)
        myServer = OSCHandler.Instance.CreateServer("myServer", inPort);
        // Set buffer size (bytes) of the server (default 1024)
        myServer.ReceiveBufferSize = 1024;
        // Set the sleeping time of the thread (default 10)
        myServer.SleepMilliseconds = 10;

    }


    void Update()
    {       
    }

}
Open Max UDP patch and modify the ip address where your OSC server is running.

Max UDP Settings

As you can see I’m running my OSC Server on 192.168.0.197 and in the code above, I specified the port to be 57121. You can modify this in the Unity inspector as well. In order to capture the OSC message and translate it into something meaningful, we need to update our Envelop class.

Modify Envelop.cs, add array for inputs.
//Class variable
GameObject[] inputs = new GameObject[NUM_INPUTS];

//Input models function
inputs[x] = sphere;
Modify Envelop.cs, add oscReceive function.
// Process OSC message
    private void receivedOSC(OSCPacket pckt)
    {
        if (pckt == null) { Debug.Log("Empty packet"); return; }

        char[] delimiterChars = { '/' };

        // Address
        string address = pckt.Address.Substring(1);

        if(address == "bundle") {
            foreach(OSCMessage message in pckt.Data) {

                Debug.Log(message.Address);

                if(message.Address.Substring(1,6) == "source") {
                    String[] packetSplit = message.Address.Split(delimiterChars);
                    int inputNumber = Int32.Parse(packetSplit[2]);

                    Debug.Log("0::" + message.Data[0]);
                    Debug.Log("1::" + message.Data[1]);
                    Debug.Log("2::" + message.Data[2]);

                    float positionX = Midway.cx + float.Parse(message.Data[0].ToString()) * Midway.xRange / 2;
                    float positionY = Midway.cy + float.Parse((string)message.Data[2].ToString()) * Midway.yRange / 2;
                    float positionZ = Midway.cz + float.Parse((string)message.Data[1].ToString()) * -Midway.zRange / 2;

                    inputs[inputNumber - 1].transform.position = new Vector3(positionX, positionY, positionZ);

                }
            }
        }
    }
Modify Envelop.cs, add to update() loop.
// Update is called once per frame
    void Update () {

        // Reads all the messages received between the previous update and this one
        // Read received messages
        for (var i = 0; i < OSCHandler.Instance.packets.Count; i++)
        {
            // Process OSC
            receivedOSC(OSCHandler.Instance.packets[i]);
            // Remove them once they have been read.
            OSCHandler.Instance.packets.Remove(OSCHandler.Instance.packets[i]);
            i--;
        }        
    }

And that’s it. If you did everything correctly, the spheres in Unity should be moving with the parameters set in Ableton.

Thanks for reading.

=.+.+.=

Divination

14 August 2017