Compiling helpful how-tos & code snippets for Lens Studio 5

## Connect SIK Interaction to Behaviour.js

// @input Component.ScriptComponent interactable
// @input string behaviorEventName

script.interactable.onTriggerEnd.add(() => {
    global.behaviorSystem.sendCustomTrigger(script.behaviorEventName);
});

## Expose a variable or function in Javascript

#### MyScript.js

script.numberVal = 1;
script.helloWorld = function () {
  print('hello world!');
};

Now you can access `numberVal` or `helloWorld` from other scripts

## Connect SIK Button to exposed Javascript function

#### Inspector Panel

1. Click Button object in Scene
2. Find PinchButton.ts component
3. Toggle Edit Event Callbacks to ON
4. Drag your custom script component into Custom Function
5. Type name of `script.functionName` as `functionName` in Value 0

## Access exposed Javascript from another Script

#### MyScript2.js

// @input SceneObject objWithScript

var myScript = script.objWithScript.getComponent("Component.ScriptComponent");

print(myScript.numberVal);
myScript.helloWorld();

#### Console output

>> 1
>> hello world!

## Expose a variable of function globally in Javascript

#### globals.js

global.numberVal = 1;
global.helloWorld = function () {
    print("hello world!");
}

#### myScript.js

print(global.numberVal)
global.helloWorld();

#### Console output

>> 1
>> hello world!

## Script Inputs in Javascript
Inputs lets you assign Assets or Components in a script via Inspector, allowing you to connect functionality across your project

#### MyScript.js

// @input SceneObject someSceneObj
// @input Component.ScriptComponent someScript
// @input Asset.Texture someTexture

print(script.someSceneObj.name);
print(script.someScript);
print(script.someTexture.getHeight());

Inputs show up in Inspector and should be filled in

## Script Inputs in TypeScript
Inputs lets you assign Assets or Components in a script via Inspector, allowing you to connect functionality across your project

#### MyScript.ts

@component
export class MyScript extends BaseScriptComponent {
  @input
  someSceneObj: SceneObject;

  @input
  someScript: ScriptComponent;

  @input
  someTexture: Texture;

  onAwake() {
      print(this.someSceneObj.name);
      print(this.someScript);
      print(this.someTexture.getHeight());
  }
}

Inputs show up in Inspector and should be filled in

## Accessing Javascript from Javascript

#### MyScript2.js

//@input Component.ScriptComponent refScript
print(script.refScript.numberVal);
script.refScript.printHelloWorld();

## Accessing Javascript from TypeScript

#### MyScript.js

script.numberVal = 1;
script.printHelloWorld = function () {
  print('hello');
};

#### MyScript_Declaration.ts

FYI: Do not put this script in your scene or you will generate a `prototype` error

export interface MyScript extends ScriptComponent {
  numberVal: number;
  printHelloWorld: () => void;
}

#### MyTypeScript.ts

import { MyScript } from './MyScript_Declaration';

@component
export class MyTypeScript extends BaseScriptComponent {
  @input('Component.ScriptComponent')
  refScript: MyScript; //this connects to MyScript.js

  onAwake() {
    print(this.refScript.numberVal);
    this.refScript.printHelloWorld();
  }
}

## Accessing TypeScript from JavaScript

#### MyTypeScript.ts

@component
export class MyTypeScript extends BaseScriptComponent {
  numberVal: number = 1;

  onAwake() {}
  printHelloWorld() {
    print('Hello, world!');
  }
}

#### MyScript.js

//@input Component.ScriptComponent refScript
print(script.refScript.numberVal);
script.refScript.printHelloWorld();

## Accessing TypeScript form TypeScript

#### MyTypeScript.ts

@component
export class MyTypeScript extends BaseScriptComponent {
  numberVal: number = 1;

  onAwake() {}
  printHelloWorld() {
    print('Hello, world!');
  }
}

#### MyTypeScript2.ts

import { MyTypeScript } from './MyTypeScript';

@component
export class MyTypeScript2 extends BaseScriptComponent {
  @input
  refScript: MyTypeScript;

  onAwake() {
    print(this.refScript.numberVal);
    this.refScript.printHelloWorld();
  }
}

## Access SIK in JavaScript

var SIK = require("SpectaclesInteractionKit/SIK").SIK;
var handInputData = SIK.HandInputData;

var rightHand = handInputData.getHand("right");
var leftHand = handInputData.getHand("left");

if (rightHand.isTracked() || leftHand.isTracked()) {
    print("yay hands!");
}

## Access SIK in TypeScript

import { HandInteractor } from "SpectaclesInteractionKit/Core/HandInteractor/HandInteractor";
import { SIK } from "SpectaclesInteractionKit/SIK";

@component
export class mySIKScript extends BaseScriptComponent {
  
    private handInputData = SIK.HandInputData;
    private rightHand = this.handInputData.getHand("right");
    private leftHand = this.handInputData.getHand("left");
    
    onAwake() {
        if (this.rightHand.isTracked() || this.leftHand.isTracked()) {
            print("yay hands!");
        }
    }
}

## Access your own Remote API in Javascript

**Prerequisites:**
1. Asset Browser > Add (+) > Import Remote Service Module
2. Set spec id on Remote Service Module Asset to random text
3. Preview -> Gear Icon > Set device type to Spectacles 2024
4. Asset Browser > Add (+) > New Javascript Script
5. Copy and paste the code below into your new script and save
6. Make sure to add the Remote Service Module as input in your Script in Inspector

// @input Asset.RemoteServiceModule rsm
const request = RemoteServiceHttpRequest.create();
request.url = "YOUR_REMOTE_API.COM";
request.method = RemoteServiceHttpRequest.HttpRequestMethod.Get;

function onResponse(response){
    print(“Status code: ” + response.statusCode);
    print(response.body);
}

script.rsm.performHttpRequest(request, onResponse);

FYI: localhost does not work and you will need to access localhost through a hosted proxy or use an actual https remote API

## Load a Remote Texture

//@input Asset.RemoteServiceModule rsm
//@input Asset.RemoteMediaModule rmm
//@input Component.Image image

let request = RemoteServiceHttpRequest.create();
request.url = "YOUR_IMAGE_URL_WITH_EXTENSION";

script.rsm
    .performHttpRequest(request, function(response){
        const dynamicResource = response.asResource()

        script.rmm.loadResourceAsImageTexture(dynamicResource, function(texture) {

            script.image.mainPass.baseTex = texture
        },function(error){
            print(error)
        })
    })

## New! Get Device Camera Texture (TypeScript)

import { arrayToBase64 } from "./Base64"

@component
export class CameraImage extends BaseScriptComponent {
    private cameraModule = require("LensStudio:CameraModule") as CameraModule

    captureFrame() {
        print("capturing frame")

        const request = CameraModule.createCameraRequest()
        request.cameraId = CameraModule.CameraId.Default_Color
        request.imageSmallerDimension = 756

        const cameraTexture = this.cameraModule.requestCamera(request)
        const provider = cameraTexture.control as CameraTextureProvider

        const eventRegistration = provider.onNewFrame.add(() => {
            provider.onNewFrame.remove(eventRegistration)

            const width = cameraTexture.getWidth() // 1008
            const height = cameraTexture.getHeight() // 756
            const readableTexture = ProceduralTextureProvider.createFromTexture(cameraTexture)
            const readableProvider = readableTexture.control as ProceduralTextureProvider
            const data = new Uint8Array(width * height * 4)
            readableProvider.getPixels(0, 0, width, height, data)
            this.encodeImage(data, width, height)

            const imageComponent = this.sceneObject.getComponent("Image")
            imageComponent.enabled = true
            imageComponent.mainPass.baseTex = readableTexture
        })
    }

    encodeImage(data: Uint8Array, width: number, height: number) {
        print(`encodeImage: ${data.length} bytes`)
        //const str = arrayToBase64(data)
        //print("encoded string length: " + str.length)
        //print(str.substring(0, 1000))
    }
}

## Alternative: Access Device Camera in Javascript (probably broken)

// @input Asset.Image displayImage

let cameraModule = require('LensStudio:CameraModule');

script.createEvent("OnStartEvent").bind(function() {
    let cameraRequest = CameraModule.createCameraRequest();
    cameraRequest.id = CameraModule.CameraId.Left_Color;
    
    let cameraTexture = cameraModule.requestCamera(cameraRequest);  
    script.displayImage.mainPass.baseTex = cameraTexture
})

## Alternative 2: Camera Texture not available? It's a bug!

Device Camera Texture unavailable or returning with width and height of 0? Here's a hack:

1. Scene Hierarchy > Add (+) > Sphere
2. Make Sphere a child of the Camera object
3. Change Sphere position to 0, 0, 100
4. Select Sphere > Inspector Panel > Right click Material slot > Highlight
5. Toggle on Base Texture and click Texture slot
6. Choose Device Camera Texture from Assets
7. Scale Sphere to scale of 1, 1, 1

This will force the device camera texture to load giving you access in your scripts

## Plug Device Camera Texture Into SnapML

// @input Component.MLComponent mlComponent
// @input SceneObject image

let cameraModule = require('LensStudio:CameraModule');
let createCameraTexture = () => {
    let request = CameraModule.createCameraRequest();
    request.cameraId = CameraModule.CameraId.Left_Color;
    return cameraModule.requestCamera(request);
}

let cameraTexControl;
let runInference = (frame) => {
    script.mlComponent.runImmediate(true);
    // Use frame.timestampMillis if you need to use wtimestamps
    script.image.enabled = true;
}

script.mlComponent.onLoadingFinished = () => {
    let cameraTex = createCameraTexture();
    cameraTexControl = cameraTex.control;
    script.mlComponent.getInput('input').texture = cameraTex;
    cameraTexControl.onNewFrame.add(runInference)
    
    let cameraInfo = 
        global.deviceInfoSystem.getTrackingCameraForId(CameraModule.CameraId.Left_Color);
    print(cameraInfo);
}

script.mlComponent.build([])

## Convert Texture Into Data

function textureToData(tex) {
    var width = tex.getWidth();
    var height = tex.getHeight();
    
    var data = new Uint8Array(width * height);

    tex.control.getPixels(0, 0, width, height, data);
    return data;
}

## Convert Data Into Texture

function dataToTexture(data, tex) {
    var width = tex.getWidth();
    var height = tex.getHeight();
    
    if (!tex) {
        tex = ProceduralTextureProvider.create(width, height, Colorspace.R);
    }
    tex.control.setPixels(0, 0, width, height, data);
    return tex;
}

## Convert Uint8Array into Base64 (to upload image)

export function arrayToBase64(bytes: Uint8Array): string {
    const encodings = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
    let base64 = ""

    var byteRemainder = bytes.length % 3
    var mainLength = bytes.length - byteRemainder

    // Main loop deals with bytes in chunks of 3
    for (var i = 0; i < mainLength; i = i + 3) {
        // Combine the three bytes into a single integer
        var chunk = (bytes[i] << 16) | (bytes[i + 1] << 8) | bytes[i + 2]

        // Use bitmasks to extract 6-bit segments from the triplet
        var a = (chunk & 16515072) >> 18 // 16515072 = (2^6 - 1) << 18
        var b = (chunk & 258048) >> 12 // 258048   = (2^6 - 1) << 12
        var c = (chunk & 4032) >> 6 // 4032     = (2^6 - 1) << 6
        var d = chunk & 63               // 63       = 2^6 - 1

        // Convert the raw binary segments to the appropriate ASCII encoding
        base64 += encodings[a] + encodings[b] + encodings[c] + encodings[d]
    }

    // Deal with the remaining bytes and padding
    if (byteRemainder === 1) {
        chunk = bytes[mainLength]

        const a = (chunk & 252) >> 2 // 252 = (2^6 - 1) << 2

        // Set the 4 least significant bits to zero
        const b = (chunk & 3) << 4 // 3   = 2^2 - 1

        base64 += encodings[a] + encodings[b] + '=='
    } else if (byteRemainder === 2) {
        chunk = (bytes[mainLength] << 8) | bytes[mainLength + 1]

        a = (chunk & 64512) >> 10 // 64512 = (2^6 - 1) << 10
        b = (chunk & 1008) >> 4 // 1008  = (2^6 - 1) << 4

        // Set the 2 least significant bits to zero
        c = (chunk & 15) << 2 // 15    = 2^4 - 1

        base64 += encodings[a] + encodings[b] + encodings[c] + '='
    }

    return base64
}

## Bonus: Convert Texture Into Grayscale

function textureToData(texture) {
    var width = texture.getWidth();
    var height = texture.getHeight();
    
    var data = new Uint8Array(width * height);

    TensorMath.textureToGrayscale(texture, data, new vec3(width, height, 1));
    return data;
}

## TypeScript - SIK Workshop Code

import { HandInteractor } from "SpectaclesInteractionKit/Core/HandInteractor/HandInteractor";
import { SIK } from "SpectaclesInteractionKit/SIK";

@component
export class PinchToSpawn extends BaseScriptComponent {
  @input
  objectToSpawn: ObjectPrefab;

  @input
  leftHandInteractor: HandInteractor;

  @input
  rightHandInteractor: HandInteractor;

  private handInputData = SIK.HandInputData;
  private leftHand = this.handInputData.getHand("left");
    private rightHand = this.handInputData.getHand("right");
    
    private spawnedObjects: SceneObject [] = []

  onAwake() {
    this.leftHand.onPinchUp(() => {
      if (this.leftHandInteractor.targetHitInfo === null) {
        this.spawnObject(this.leftHand.indexTip.position);
      }
    });

    this.rightHand.onPinchUp(() => {
      if (this.rightHandInteractor.targetHitInfo === null) {
        this.spawnObject(this.rightHand.indexTip.position);
      }
    });
  }

  private spawnObject(position: vec3): void {
    const spawnedObject = this.objectToSpawn.instantiate(this.getSceneObject());
      spawnedObject.getTransform().setWorldPosition(position);
      
      this.spawnedObjects.push(spawnedObject);
  }
    
clearAll() {
      this.spawnedObjects.forEach((obj) => {
          obj.destroy();
      });
    this.spawnedObjects = []
  }
}