http://localhost:3001/agents
http://localhost:3001/AGENT_ID/hyperfy
b850bc30-45f8-0041-a00a-83df46d8555d
http://localhost:3001/b850bc30-45f8-0041-a00a-83df46d8555d/hyperfy
First, let's look at how the states are defined. The code initializes state timing constants at the very beginning:
const BUBBLE_TIME = 5 // Speaking state duration
const EMOTE_TIME = 2 // Emoting state duration
const LOOK_TIME = 5 // Looking state duration
The client-side FSM maintains its state through the data object:
const data = {} // Empty object to track current states
The FSM has three main state handlers, each triggered by specific events:
app.on('say', value => {
data.say = { timer: 0 } // Initialize speaking state
nametag.active = false // Hide nametag
bubbleText.value = value // Set speech content
bubble.active = true // Show speech bubble
})
app.on('emote', url => {
data.emote = { timer: 0 } // Initialize emoting state
vrm.setEmote(url) // Play animation
})
app.on('look', playerId => {
data.look = { playerId, timer: 0 } // Initialize looking state
})
The state transitions are managed in the update loop, which checks and updates all active states:
app.on('update', delta => {
// Speaking state management
if (data.say) {
data.say.timer += delta
if (data.say.timer > BUBBLE_TIME) {
data.say = null // Reset state
bubble.active = false // Hide bubble
nametag.active = true // Show nametag
}
}
// Emoting state management
if (data.emote) {
data.emote.timer += delta
if (data.emote.timer > EMOTE_TIME) {
data.emote = null // Reset state
vrm.setEmote(idleEmoteUrl) // Return to idle animation
}
}
// Looking state management
if (data.look) {
const player = world.getPlayer(data.look.playerId)
if (player) {
// Calculate direction and rotate agent
const direction = v1.copy(player.position).sub(vrm.position)
direction.y = 0
const angle = Math.atan2(direction.x, direction.z) + Math.PI
vrm.quaternion.setFromAxisAngle(UP, angle)
}
data.look.timer += delta
if (data.look.timer > LOOK_TIME) {
data.look = null // Reset state
}
}
})
The server side of the FSM handles event detection and command generation. Here's how it processes events:
world.on('enter', player => {
info.events.push({
type: 'player-enter',
playerId: player.entityId,
})
changed = true
})
world.on('chat', msg => {
if (msg.fromId === app.instanceId) return
if(!msg.body) return
info.events.push({
type: 'chat',
...msg,
})
if (info.events.length > 16) {
info.events.shift()
}
changed = true
})
When events occur, the server communicates with an external webhook to determine state changes:
async function notify() {
if (!config.url) return
changed = false
notifying = true
try {
const resp = await fetch(config.url, {
headers: {
'Content-Type': 'application/json',
},
method: 'POST',
body: JSON.stringify(info),
})
data = await resp.json()
} catch (err) {
console.error('notify failed')
}
// Process webhook response and trigger state changes
if (data.say && data.say != "null") {
app.send('say', data.say)
}
if (data.emote) {
app.send('emote', emoteUrls[data.emote])
}
if (data.look) {
app.send('look', data.look)
}
}
This implementation demonstrates key FSM characteristics:
Code:
const BUBBLE_TIME = 5
const EMOTE_TIME = 2
const LOOK_TIME = 5
const UP = new Vector3(0, 1, 0)
const v1 = new Vector3()
const v2 = new Vector3()
const v3 = new Vector3()
const q1 = new Quaternion()
const q2 = new Quaternion()
const m1 = new Matrix4()
const vrm = app.get('avatar')
// SERVER
if (world.isServer) {
const config = app.config
// send initial state
const state = {
ready: true,
}
app.state = state
app.send('state', state)
// spawn controller
const ctrl = app.create('controller')
ctrl.position.copy(app.position)
world.add(ctrl)
ctrl.quaternion.copy(app.quaternion)
ctrl.add(vrm)
// read emotes
const emoteUrls = {}
if (config.emote1Name && config.emote1?.url) {
emoteUrls[config.emote1Name] = config.emote1.url
}
if (config.emote2Name && config.emote2?.url) {
emoteUrls[config.emote2Name] = config.emote2.url
}
if (config.emote3Name && config.emote3?.url) {
emoteUrls[config.emote3Name] = config.emote3.url
}
if (config.emote4Name && config.emote4?.url) {
emoteUrls[config.emote4Name] = config.emote4.url
}
// observe environment
let changed = true
let notifying = false
const info = {
world: {
id: null, // todo
name: null, // todo
url: null, // todo
context: config.context || 'You are in a virtual world powered by Hyperfy',
},
you: {
id: app.instanceId,
name: config.name,
},
emotes: Object.keys(emoteUrls),
triggers: [],
events: [],
}
world.on('enter', player => {
info.events.push({
type: 'player-enter',
playerId: player.entityId,
})
changed = true
})
world.on('leave', player => {
info.events.push({
type: 'player-leave',
playerId: player.entityId,
})
changed = true
})
world.on('chat', msg => {
if (msg.fromId === app.instanceId) return
if(!msg.body) return
info.events.push({
type: 'chat',
...msg,
})
if (info.events.length > 16) {
info.events.shift()
}
changed = true
})
// DEBUG
// app.send('say', 'Test!')
// app.send('emote', emoteUrls.wave)
async function notify() {
if (!config.url) return
changed = false
notifying = true
console.log('notifying...', info)
let data
try {
const resp = await fetch(config.url, {
headers: {
'Content-Type': 'application/json',
},
method: 'POST',
body: JSON.stringify(info),
})
data = await resp.json()
} catch (err) {
console.error('notify failed')
}
notifying = false
if (!data) return
console.log(data)
if (data.say && data.say != "null") {
app.send('say', data.say)
const msg = {
id: uuid(),
from: config.name + ' (agent)',
fromId: app.instanceId,
body: data.say,
createdAt: world.getTimestamp(),
}
world.chat(msg, true)
info.events.push({
type: 'chat',
...msg,
})
}
if (data.emote) {
const url = emoteUrls[data.emote]
app.send('emote', url)
}
if (data.look) {
app.send('look', data.look)
}
}
app.on('fixedUpdate', delta => {
if (changed && !notifying) {
notify()
}
// const v1 = new Vector3(0,0,1)
// app.on('fixedUpdate', delta => {
// ctrl.move(v1.set(0,0,1).multiplyScalar(1 * delta))
// })
})
}
// CLIENT
if (world.isClient) {
const config = app.config
const idleEmoteUrl = config.emote0?.url
world.attach(vrm)
let state = app.state
if (state.ready) {
init()
} else {
world.remove(vrm)
app.on('state', _state => {
state = _state
init()
})
}
// setup bubble
const bubble = app.create('ui')
bubble.width = 300
bubble.height = 512
bubble.size = 0.005
bubble.pivot = 'bottom-center'
bubble.billboard = 'full'
bubble.justifyContent = 'flex-end'
bubble.alignItems = 'center'
bubble.position.y = 2
bubble.active = false
const bubbleBox = app.create('uiview')
bubbleBox.backgroundColor = 'rgba(0, 0, 0, 0.95)'
bubbleBox.borderRadius = 20
bubbleBox.padding = 20
bubble.add(bubbleBox)
const bubbleText = app.create('uitext')
bubbleText.color = 'white'
bubbleText.fontWeight = 100
bubbleText.lineHeight = 1.4
bubbleText.fontSize = 16
bubbleText.value = '...'
bubbleBox.add(bubbleText)
vrm.add(bubble)
// setup nametag
const nametag = app.create('nametag')
nametag.label = config.name
nametag.position.y = 2
vrm.add(nametag)
function init() {
world.add(vrm)
vrm.setEmote(idleEmoteUrl)
}
const data = {}
app.on('say', value => {
data.say = { timer: 0 }
nametag.active = false
bubbleText.value = value
bubble.active = true
})
app.on('emote', url => {
data.emote = { timer: 0 }
vrm.setEmote(url)
})
app.on('look', playerId => {
data.look = { playerId, timer: 0 }
})
app.on('update', delta => {
if (data.say) {
data.say.timer += delta
if (data.say.timer > BUBBLE_TIME) {
data.say = null
bubble.active = false
nametag.active = true
}
}
if (data.emote) {
data.emote.timer += delta
if (data.emote.timer > EMOTE_TIME) {
data.emote = null
vrm.setEmote(idleEmoteUrl)
}
}
if (data.look) {
const player = world.getPlayer(data.look.playerId)
if (player) {
const direction = v1.copy(player.position).sub(vrm.position)
direction.y = 0
const angle = Math.atan2(direction.x, direction.z) + Math.PI
vrm.quaternion.setFromAxisAngle(UP, angle)
}
data.look.timer += delta
if (data.look.timer > LOOK_TIME) {
data.look = null
}
}
})
}
// CONFIG
app.configure(() => {
return [
{
key: 'name',
type: 'text',
label: 'Name',
},
{
key: 'context',
type: 'textarea',
label: 'Context',
},
{
key: 'url',
type: 'text',
label: 'URL',
},
{
key: 'emotes',
type: 'section',
label: 'Emotes',
},
{
key: 'emote0',
type: 'file',
label: 'Idle',
kind: 'emote',
},
{
key: 'emote',
type: 'switch',
label: 'Custom',
options: [
{ label: '1', value: '1' },
{ label: '2', value: '2' },
{ label: '3', value: '3' },
{ label: '4', value: '4' },
],
},
...customEmoteFields('1'),
...customEmoteFields('2'),
...customEmoteFields('3'),
...customEmoteFields('4'),
]
function customEmoteFields(n) {
return [
{
key: `emote${n}Name`,
type: 'text',
label: 'Name',
when: [{ key: 'emote', op: 'eq', value: n }],
},
{
key: `emote${n}`,
type: 'file',
label: 'Emote',
kind: 'emote',
when: [{ key: 'emote', op: 'eq', value: n }],
},
]
}
})
Let me help explain how behavior trees represent an evolution from finite state machines, breaking this down step by step with clear examples from our code.
First, let's understand how behavior trees differ fundamentally from FSMs:
In an FSM, we directly handle state transitions through events:
Behavior trees instead use a hierarchical approach:
Let's examine the fundamental components that make behavior trees work:
// Base node that all others extend from
class BehaviorNode {
constructor(name) {
this.name = name
}
tick(context) { } // Abstract method each node implements
}
// Runs children in sequence until one fails
class Sequence extends BehaviorNode {
tick(context) {
for (const child of this.children) {
if (child.tick(context) !== 'SUCCESS') {
return 'FAILURE'
}
}
return 'SUCCESS'
}
}
// Checks if a behavior can run
class ConditionNode extends BehaviorNode {
tick(context) {
return this.condition(context) ? 'SUCCESS' : 'FAILURE'
}
}
// Performs actual behavior
class ActionNode extends BehaviorNode {
tick(context) {
return this.action(context)
}
}
Here's how the behavior tree processes each frame:
Let's look at how we handle chat interactions in both paradigms:
app.on('say', value => {
data.say = { timer: 0 }
nametag.active = false
bubbleText.value = value
bubble.active = true
})
// Behavior Tree Style - Modular components
new Sequence('chatHandler', [
// First check if we can chat
new ConditionNode('hasPendingChat', () =>
pendingChat !== null
),
// Then check distance
new ConditionNode('isInRange', context =>
context.playerDistance <= 2
),
// Finally handle the chat
new ActionNode('handleChat', () => {
showBubble(pendingChat.message)
sendChat(pendingChat.message)
pendingChat = null
return 'SUCCESS'
})
])
The real power comes from combining nodes for complex behaviors:
const behaviorTree = new Selector('root', [
new Sequence('closeInteraction', [
// Only runs for nearby players
new ConditionNode('isVeryClose', context =>
context.playerDistance <= 2
),
// Handles multiple behaviors in sequence
new ActionNode('greet', context => {
if (currentZone !== 'close') {
currentZone = 'close'
data.emote = { timer: 0, type: 'bow' }
vrm.setEmote(config.emote1?.url)
showBubble("Perfect! Now we can talk.")
}
updateRotation(context)
return 'SUCCESS'
})
])
])
Modularity
Readability
Maintainability
Flexibility
const BUBBLE_TIME = 5
const EMOTE_TIME = 2
const LOOK_TIME = 5
const UP = new Vector3(0, 1, 0)
const v1 = new Vector3()
const vrm = app.get('avatar')
// SERVER
if (world.isServer) {
const config = app.config
const state = { ready: true }
app.state = state
app.send('state', state)
const ctrl = app.create('controller')
ctrl.position.copy(app.position)
world.add(ctrl)
ctrl.quaternion.copy(app.quaternion)
ctrl.add(vrm)
const emoteUrls = {}
if (config.emote1Name && config.emote1?.url) {
emoteUrls[config.emote1Name] = config.emote1.url
}
if (config.emote2Name && config.emote2?.url) {
emoteUrls[config.emote2Name] = config.emote2.url
}
if (config.emote3Name && config.emote3?.url) {
emoteUrls[config.emote3Name] = config.emote3.url
}
if (config.emote4Name && config.emote4?.url) {
emoteUrls[config.emote4Name] = config.emote4.url
}
let changed = true
let notifying = false
const info = {
world: {
id: null,
name: null,
url: null,
context: config.context || 'You are in a virtual world powered by Hyperfy'
},
you: {
id: app.instanceId,
name: config.name,
},
emotes: Object.keys(emoteUrls),
triggers: [],
events: [],
}
world.on('enter', player => {
info.events.push({
type: 'player-enter',
playerId: player.id
})
changed = true
})
world.on('leave', player => {
info.events.push({
type: 'player-leave',
playerId: player.id
})
changed = true
})
world.on('chat', msg => {
if (msg.fromId === app.instanceId) return
if (msg.skipBehavior) return
const player = world.getPlayer(msg.fromId)
if (!player) return
// Add distance check before processing any chat events
const distance = player.position.distanceTo(ctrl.position)
if (distance <= 2) {
info.events.push({ type: 'chat', ...msg })
changed = true
}
})
async function notify() {
if (!config.url || notifying) return
changed = false
notifying = true
try {
const resp = await fetch(config.url, {
headers: { 'Content-Type': 'application/json' },
method: 'POST',
body: JSON.stringify(info)
})
const data = await resp.json()
if (data?.say) app.send('say', data.say)
if (data?.emote) app.send('emote', emoteUrls[data.emote])
if (data?.look) app.send('look', data.look)
} catch (err) {
console.error('notify failed:', err)
}
notifying = false
}
app.on('fixedUpdate', delta => {
if (changed && !notifying) notify()
})
}
// CLIENT
if (world.isClient) {
class BehaviorNode {
constructor(name) {
this.name = name
}
tick(context) { }
}
class Selector extends BehaviorNode {
constructor(name, children) {
super(name)
this.children = children
}
tick(context) {
for (const child of this.children) {
const status = child.tick(context)
if (status === 'SUCCESS' || status === 'RUNNING') {
return status
}
}
return 'FAILURE'
}
}
class Sequence extends BehaviorNode {
constructor(name, children) {
super(name)
this.children = children
}
tick(context) {
for (const child of this.children) {
const status = child.tick(context)
if (status !== 'SUCCESS') {
return status
}
}
return 'SUCCESS'
}
}
class ConditionNode extends BehaviorNode {
constructor(name, condition) {
super(name)
this.condition = condition
}
tick(context) {
return this.condition(context) ? 'SUCCESS' : 'FAILURE'
}
}
class ActionNode extends BehaviorNode {
constructor(name, action) {
super(name)
this.action = action
}
tick(context) {
return this.action(context)
}
}
const config = app.config
const idleEmoteUrl = config.emote0?.url
let currentZone = null
let messageCounter = 0
let pendingChat = null
const farMessages = [
"I can barely see you from here!",
"You're too far away to talk!",
"Come closer if you want to chat!",
"I can see you in the distance!"
]
const mediumMessages = [
"Almost there, just a bit closer!",
"I still can't quite hear you!",
"A few more steps closer please!",
"That's better, but come a little closer!"
]
function getNextMessage(messages) {
return messages[messageCounter++ % messages.length]
}
function sendChat(message, skipBehavior = false) {
world.chat({
id: uuid(),
from: config.name + ' (agent)',
fromId: app.instanceId,
body: message,
createdAt: world.getTimestamp(),
skipBehavior
}, true)
}
function showBubble(message) {
data.say = { timer: 0 }
bubble.active = true
nametag.active = false
bubbleText.value = message
}
function init() {
world.add(vrm)
vrm.setEmote(idleEmoteUrl)
}
// UI Setup
world.attach(vrm)
let state = app.state
if (state.ready) {
init()
} else {
world.remove(vrm)
app.on('state', _state => {
state = _state
init()
})
}
const bubble = app.create('ui')
bubble.width = 300
bubble.height = 512
bubble.size = 0.005
bubble.pivot = 'bottom-center'
bubble.billboard = 'full'
bubble.justifyContent = 'flex-end'
bubble.alignItems = 'center'
bubble.position.y = 2
bubble.active = false
const bubbleBox = app.create('uiview')
bubbleBox.backgroundColor = 'rgba(0, 0, 0, 0.95)'
bubbleBox.borderRadius = 20
bubbleBox.padding = 20
bubble.add(bubbleBox)
const bubbleText = app.create('uitext')
bubbleText.color = 'white'
bubbleText.fontWeight = 100
bubbleText.lineHeight = 1.4
bubbleText.fontSize = 16
bubbleText.value = '...'
bubbleBox.add(bubbleText)
vrm.add(bubble)
const nametag = app.create('nametag')
nametag.label = config.name
nametag.position.y = 2
vrm.add(nametag)
function updateRotation(context) {
if (context.player && context.playerDistance <= 10) {
const direction = v1.copy(context.player.position).sub(vrm.position)
direction.y = 0
const angle = Math.atan2(direction.x, direction.z) + Math.PI
vrm.quaternion.setFromAxisAngle(UP, angle)
}
}
const data = {}
const behaviorTree = new Selector('root', [
new Sequence('apiChatHandler', [
new ConditionNode('hasPendingApiChat', () =>
pendingChat && typeof pendingChat === 'object' && pendingChat.fromApi
),
new ActionNode('handleApiChat', () => {
showBubble(pendingChat.message)
sendChat(pendingChat.message, true)
pendingChat = null
return 'SUCCESS'
})
]),
new Sequence('playerChatHandler', [
new ConditionNode('hasPendingPlayerChat', () =>
pendingChat && typeof pendingChat === 'string'
),
new ActionNode('handlePlayerChat', context => {
if (context.playerDistance > 10) {
sendChat("I can't hear you from that far away! Come closer if you want to talk!", true)
} else if (context.playerDistance > 2) {
sendChat("If you want to speak to me, come closer!", true)
}
pendingChat = null
return 'SUCCESS'
})
]),
new Sequence('closeInteraction', [
new ConditionNode('isVeryClose', context => context.playerDistance <= 2),
new ActionNode('greet', context => {
if (currentZone !== 'close') {
currentZone = 'close'
data.emote = { timer: 0, type: 'bow' }
if (config.emote1?.url) {
vrm.setEmote(config.emote1.url)
}
showBubble("Ah, perfect! Now we can talk properly.")
sendChat("Ah, perfect! Now we can talk properly.", true)
}
updateRotation(context)
return 'SUCCESS'
})
]),
new Sequence('mediumInteraction', [
new ConditionNode('isMediumDistance', context =>
context.playerDistance > 2 && context.playerDistance <= 5
),
new ActionNode('mediumDistance', context => {
if (currentZone !== 'medium') {
currentZone = 'medium'
data.emote = { timer: 0, type: 'wave' }
if (config.emote2?.url) {
vrm.setEmote(config.emote2.url)
}
const message = getNextMessage(mediumMessages)
showBubble(message)
sendChat(message, true)
}
updateRotation(context)
return 'SUCCESS'
})
]),
new Sequence('farInteraction', [
new ConditionNode('isFarDistance', context =>
context.playerDistance > 5 && context.playerDistance <= 10
),
new ActionNode('farDistance', context => {
if (currentZone !== 'far') {
currentZone = 'far'
const message = getNextMessage(farMessages)
showBubble(message)
sendChat(message, true)
}
updateRotation(context)
return 'SUCCESS'
})
]),
new ActionNode('idle', context => {
if (currentZone !== 'idle') {
currentZone = 'idle'
if (!data.emote) {
vrm.setEmote(idleEmoteUrl)
}
bubble.active = false
nametag.active = true
}
return 'SUCCESS'
})
])
function updateContext(delta) {
const context = {
delta,
playerDistance: Infinity,
player: null
}
const player = world.getPlayer()
if (player) {
context.player = player
context.playerDistance = player.position.distanceTo(vrm.position)
}
return context
}
world.on('chat', msg => {
if (msg.fromId !== app.instanceId && !msg.skipBehavior) {
pendingChat = msg.body
}
})
app.on('say', value => {
pendingChat = { message: value, fromApi: true }
})
app.on('emote', url => {
if (currentZone !== 'idle') {
data.emote = { timer: 0 }
vrm.setEmote(url)
}
})
app.on('update', delta => {
const context = updateContext(delta)
behaviorTree.tick(context)
if (data.say?.timer !== undefined) {
data.say.timer += delta
if (data.say.timer > BUBBLE_TIME) {
data.say = null
bubble.active = false
nametag.active = true
}
}
if (data.emote?.timer !== undefined) {
data.emote.timer += delta
if (data.emote.timer > EMOTE_TIME) {
data.emote = null
vrm.setEmote(idleEmoteUrl)
}
}
})
}
// Configuration
app.configure(() => [
{ key: 'name', type: 'text', label: 'Name' },
{ key: 'context', type: 'textarea', label: 'Context' },
{ key: 'url', type: 'text', label: 'URL' },
{ key: 'emotes', type: 'section', label: 'Emotes' },
{ key: 'emote0', type: 'file', label: 'Idle', kind: 'emote' },
{
key: 'emote',
type: 'switch',
label: 'Custom',
options: [1, 2, 3, 4].map(n => ({ label: n.toString(), value: n.toString() }))
},
...[1, 2, 3, 4].flatMap(n => [
{
key: `emote${n}Name`,
type: 'text',
label: 'Name',
when: [{ key: 'emote', op: 'eq', value: n.toString() }]
},
{
key: `emote${n}`,
type: 'file',
label: 'Emote',
kind: 'emote',
when: [{ key: 'emote', op: 'eq', value: n.toString() }]
}
])
])
or
By clicking below, you agree to our terms of service.
New to HackMD? Sign up