# AI Agent Behaviors in Hyperfy
```
http://localhost:3001/agents
http://localhost:3001/AGENT_ID/hyperfy
b850bc30-45f8-0041-a00a-83df46d8555d
http://localhost:3001/b850bc30-45f8-0041-a00a-83df46d8555d/hyperfy
```
## Finite State Machine (FSM) Agent
### Core FSM States and Transitions
```mermaid
stateDiagram-v2
direction LR
state "Idle" as idle {
[*] --> ShowNametag
ShowNametag --> PlayIdleAnimation
}
state "Speaking" as speaking {
[*] --> ShowBubble
ShowBubble --> HideNametag
}
state "Emoting" as emoting {
[*] --> PlayAnimation
}
state "Looking" as looking {
[*] --> CalculateAngle
CalculateAngle --> RotateAgent
}
[*] --> idle
idle --> speaking: app.on('say')
speaking --> idle: timer > BUBBLE_TIME
idle --> emoting: app.on('emote')
emoting --> idle: timer > EMOTE_TIME
idle --> looking: app.on('look')
looking --> idle: timer > LOOK_TIME
note right of idle
Default state
Shows nametag
Plays idle animation
end note
note right of speaking
Shows speech bubble
Displays message text
Timer: 5 seconds
end note
note right of emoting
Plays custom animation
Timer: 2 seconds
end note
note right of looking
Faces target player
Timer: 5 seconds
end note
```
First, let's look at how the states are defined. The code initializes state timing constants at the very beginning:
```javascript
const BUBBLE_TIME = 5 // Speaking state duration
const EMOTE_TIME = 2 // Emoting state duration
const LOOK_TIME = 5 // Looking state duration
```
The client-side FSM maintains its state through the data object:
```javascript
const data = {} // Empty object to track current states
```
```mermaid
sequenceDiagram
participant S as Server
participant E as Event Handlers
participant D as Data Object
participant U as Update Loop
S->>E: Send Command
E->>D: Initialize Timer
D->>U: Check Active States
U->>D: Update Timers
U->>D: Reset if Expired
```
### State Handler Flow
The FSM has three main state handlers, each triggered by specific events:
1. Speaking State:
```javascript
app.on('say', value => {
data.say = { timer: 0 } // Initialize speaking state
nametag.active = false // Hide nametag
bubbleText.value = value // Set speech content
bubble.active = true // Show speech bubble
})
```
2. Emoting State:
```javascript
app.on('emote', url => {
data.emote = { timer: 0 } // Initialize emoting state
vrm.setEmote(url) // Play animation
})
```
3. Looking State:
```javascript
app.on('look', playerId => {
data.look = { playerId, timer: 0 } // Initialize looking state
})
```
The state transitions are managed in the update loop, which checks and updates all active states:
```javascript
app.on('update', delta => {
// Speaking state management
if (data.say) {
data.say.timer += delta
if (data.say.timer > BUBBLE_TIME) {
data.say = null // Reset state
bubble.active = false // Hide bubble
nametag.active = true // Show nametag
}
}
// Emoting state management
if (data.emote) {
data.emote.timer += delta
if (data.emote.timer > EMOTE_TIME) {
data.emote = null // Reset state
vrm.setEmote(idleEmoteUrl) // Return to idle animation
}
}
// Looking state management
if (data.look) {
const player = world.getPlayer(data.look.playerId)
if (player) {
// Calculate direction and rotate agent
const direction = v1.copy(player.position).sub(vrm.position)
direction.y = 0
const angle = Math.atan2(direction.x, direction.z) + Math.PI
vrm.quaternion.setFromAxisAngle(UP, angle)
}
data.look.timer += delta
if (data.look.timer > LOOK_TIME) {
data.look = null // Reset state
}
}
})
```
### Server-Client Communication Flow
```mermaid
graph TD
subgraph Server
A[Event Detection] --> B[Webhook Notification]
B --> C[Command Generation]
end
subgraph Client FSM
D[Command Reception] --> E[State Initialization]
E --> F[Visual Updates]
F --> G[Timer Management]
end
C -->|app.send| D
```
The server side of the FSM handles event detection and command generation. Here's how it processes events:
```javascript
world.on('enter', player => {
info.events.push({
type: 'player-enter',
playerId: player.entityId,
})
changed = true
})
world.on('chat', msg => {
if (msg.fromId === app.instanceId) return
if(!msg.body) return
info.events.push({
type: 'chat',
...msg,
})
if (info.events.length > 16) {
info.events.shift()
}
changed = true
})
```
When events occur, the server communicates with an external webhook to determine state changes:
```javascript
async function notify() {
if (!config.url) return
changed = false
notifying = true
try {
const resp = await fetch(config.url, {
headers: {
'Content-Type': 'application/json',
},
method: 'POST',
body: JSON.stringify(info),
})
data = await resp.json()
} catch (err) {
console.error('notify failed')
}
// Process webhook response and trigger state changes
if (data.say && data.say != "null") {
app.send('say', data.say)
}
if (data.emote) {
app.send('emote', emoteUrls[data.emote])
}
if (data.look) {
app.send('look', data.look)
}
}
```
This implementation demonstrates key FSM characteristics:
- Well-defined states (speaking, emoting, looking)
- Clear state transitions triggered by events or timers
- State-specific behaviors and visual representations
- Automatic state reset mechanisms
- Centralized state management through the data object
Code:
```javascript
const BUBBLE_TIME = 5
const EMOTE_TIME = 2
const LOOK_TIME = 5
const UP = new Vector3(0, 1, 0)
const v1 = new Vector3()
const v2 = new Vector3()
const v3 = new Vector3()
const q1 = new Quaternion()
const q2 = new Quaternion()
const m1 = new Matrix4()
const vrm = app.get('avatar')
// SERVER
if (world.isServer) {
const config = app.config
// send initial state
const state = {
ready: true,
}
app.state = state
app.send('state', state)
// spawn controller
const ctrl = app.create('controller')
ctrl.position.copy(app.position)
world.add(ctrl)
ctrl.quaternion.copy(app.quaternion)
ctrl.add(vrm)
// read emotes
const emoteUrls = {}
if (config.emote1Name && config.emote1?.url) {
emoteUrls[config.emote1Name] = config.emote1.url
}
if (config.emote2Name && config.emote2?.url) {
emoteUrls[config.emote2Name] = config.emote2.url
}
if (config.emote3Name && config.emote3?.url) {
emoteUrls[config.emote3Name] = config.emote3.url
}
if (config.emote4Name && config.emote4?.url) {
emoteUrls[config.emote4Name] = config.emote4.url
}
// observe environment
let changed = true
let notifying = false
const info = {
world: {
id: null, // todo
name: null, // todo
url: null, // todo
context: config.context || 'You are in a virtual world powered by Hyperfy',
},
you: {
id: app.instanceId,
name: config.name,
},
emotes: Object.keys(emoteUrls),
triggers: [],
events: [],
}
world.on('enter', player => {
info.events.push({
type: 'player-enter',
playerId: player.entityId,
})
changed = true
})
world.on('leave', player => {
info.events.push({
type: 'player-leave',
playerId: player.entityId,
})
changed = true
})
world.on('chat', msg => {
if (msg.fromId === app.instanceId) return
if(!msg.body) return
info.events.push({
type: 'chat',
...msg,
})
if (info.events.length > 16) {
info.events.shift()
}
changed = true
})
// DEBUG
// app.send('say', 'Test!')
// app.send('emote', emoteUrls.wave)
async function notify() {
if (!config.url) return
changed = false
notifying = true
console.log('notifying...', info)
let data
try {
const resp = await fetch(config.url, {
headers: {
'Content-Type': 'application/json',
},
method: 'POST',
body: JSON.stringify(info),
})
data = await resp.json()
} catch (err) {
console.error('notify failed')
}
notifying = false
if (!data) return
console.log(data)
if (data.say && data.say != "null") {
app.send('say', data.say)
const msg = {
id: uuid(),
from: config.name + ' (agent)',
fromId: app.instanceId,
body: data.say,
createdAt: world.getTimestamp(),
}
world.chat(msg, true)
info.events.push({
type: 'chat',
...msg,
})
}
if (data.emote) {
const url = emoteUrls[data.emote]
app.send('emote', url)
}
if (data.look) {
app.send('look', data.look)
}
}
app.on('fixedUpdate', delta => {
if (changed && !notifying) {
notify()
}
// const v1 = new Vector3(0,0,1)
// app.on('fixedUpdate', delta => {
// ctrl.move(v1.set(0,0,1).multiplyScalar(1 * delta))
// })
})
}
// CLIENT
if (world.isClient) {
const config = app.config
const idleEmoteUrl = config.emote0?.url
world.attach(vrm)
let state = app.state
if (state.ready) {
init()
} else {
world.remove(vrm)
app.on('state', _state => {
state = _state
init()
})
}
// setup bubble
const bubble = app.create('ui')
bubble.width = 300
bubble.height = 512
bubble.size = 0.005
bubble.pivot = 'bottom-center'
bubble.billboard = 'full'
bubble.justifyContent = 'flex-end'
bubble.alignItems = 'center'
bubble.position.y = 2
bubble.active = false
const bubbleBox = app.create('uiview')
bubbleBox.backgroundColor = 'rgba(0, 0, 0, 0.95)'
bubbleBox.borderRadius = 20
bubbleBox.padding = 20
bubble.add(bubbleBox)
const bubbleText = app.create('uitext')
bubbleText.color = 'white'
bubbleText.fontWeight = 100
bubbleText.lineHeight = 1.4
bubbleText.fontSize = 16
bubbleText.value = '...'
bubbleBox.add(bubbleText)
vrm.add(bubble)
// setup nametag
const nametag = app.create('nametag')
nametag.label = config.name
nametag.position.y = 2
vrm.add(nametag)
function init() {
world.add(vrm)
vrm.setEmote(idleEmoteUrl)
}
const data = {}
app.on('say', value => {
data.say = { timer: 0 }
nametag.active = false
bubbleText.value = value
bubble.active = true
})
app.on('emote', url => {
data.emote = { timer: 0 }
vrm.setEmote(url)
})
app.on('look', playerId => {
data.look = { playerId, timer: 0 }
})
app.on('update', delta => {
if (data.say) {
data.say.timer += delta
if (data.say.timer > BUBBLE_TIME) {
data.say = null
bubble.active = false
nametag.active = true
}
}
if (data.emote) {
data.emote.timer += delta
if (data.emote.timer > EMOTE_TIME) {
data.emote = null
vrm.setEmote(idleEmoteUrl)
}
}
if (data.look) {
const player = world.getPlayer(data.look.playerId)
if (player) {
const direction = v1.copy(player.position).sub(vrm.position)
direction.y = 0
const angle = Math.atan2(direction.x, direction.z) + Math.PI
vrm.quaternion.setFromAxisAngle(UP, angle)
}
data.look.timer += delta
if (data.look.timer > LOOK_TIME) {
data.look = null
}
}
})
}
// CONFIG
app.configure(() => {
return [
{
key: 'name',
type: 'text',
label: 'Name',
},
{
key: 'context',
type: 'textarea',
label: 'Context',
},
{
key: 'url',
type: 'text',
label: 'URL',
},
{
key: 'emotes',
type: 'section',
label: 'Emotes',
},
{
key: 'emote0',
type: 'file',
label: 'Idle',
kind: 'emote',
},
{
key: 'emote',
type: 'switch',
label: 'Custom',
options: [
{ label: '1', value: '1' },
{ label: '2', value: '2' },
{ label: '3', value: '3' },
{ label: '4', value: '4' },
],
},
...customEmoteFields('1'),
...customEmoteFields('2'),
...customEmoteFields('3'),
...customEmoteFields('4'),
]
function customEmoteFields(n) {
return [
{
key: `emote${n}Name`,
type: 'text',
label: 'Name',
when: [{ key: 'emote', op: 'eq', value: n }],
},
{
key: `emote${n}`,
type: 'file',
label: 'Emote',
kind: 'emote',
when: [{ key: 'emote', op: 'eq', value: n }],
},
]
}
})
```
## Behavior Tree Agent
Let me help explain how behavior trees represent an evolution from finite state machines, breaking this down step by step with clear examples from our code.
### Core Architecture
First, let's understand how behavior trees differ fundamentally from FSMs:
In an FSM, we directly handle state transitions through events:
- Each state is isolated
- Transitions are explicitly defined
- States can't overlap
- Adding states increases complexity exponentially
Behavior trees instead use a hierarchical approach:
- Behaviors are organized in a tree structure
- Multiple behaviors can be active
- Complex behaviors are built from simple ones
- Adding behaviors is modular and scalable
```mermaid
stateDiagram-v2
direction LR
[*] --> Idle
Idle --> Speaking: say event
Speaking --> Idle: timer
Idle --> Emoting: emote event
Emoting --> Idle: timer
note right of Idle
FSM Architecture
Direct state transitions
end note
```
```mermaid
graph TD
subgraph "Behavior Tree Architecture"
Root[Root Selector] --> Chat[Chat Sequence]
Root --> Zones[Zone Sequence]
Chat --> C1[Check Chat]
Chat --> C2[Handle Chat]
Zones --> Z1[Check Distance]
Zones --> Z2[Handle Zone]
end
```
### Core Building Blocks
Let's examine the fundamental components that make behavior trees work:
```javascript
// Base node that all others extend from
class BehaviorNode {
constructor(name) {
this.name = name
}
tick(context) { } // Abstract method each node implements
}
// Runs children in sequence until one fails
class Sequence extends BehaviorNode {
tick(context) {
for (const child of this.children) {
if (child.tick(context) !== 'SUCCESS') {
return 'FAILURE'
}
}
return 'SUCCESS'
}
}
// Checks if a behavior can run
class ConditionNode extends BehaviorNode {
tick(context) {
return this.condition(context) ? 'SUCCESS' : 'FAILURE'
}
}
// Performs actual behavior
class ActionNode extends BehaviorNode {
tick(context) {
return this.action(context)
}
}
```
### Execution Flow
Here's how the behavior tree processes each frame:
```mermaid
sequenceDiagram
participant C as Context Update
participant R as Root Node
participant S as Sequences
participant A as Actions
C->>R: New frame starts
R->>S: Check conditions
S->>A: Run valid actions
A-->>R: Return status
R-->>C: Update complete
```
### Practical Example
Let's look at how we handle chat interactions in both paradigms:
#### FSM Style - Direct event handler
```javascript
app.on('say', value => {
data.say = { timer: 0 }
nametag.active = false
bubbleText.value = value
bubble.active = true
})
```
#### Behavior Tree - Modular Components
```javascript
// Behavior Tree Style - Modular components
new Sequence('chatHandler', [
// First check if we can chat
new ConditionNode('hasPendingChat', () =>
pendingChat !== null
),
// Then check distance
new ConditionNode('isInRange', context =>
context.playerDistance <= 2
),
// Finally handle the chat
new ActionNode('handleChat', () => {
showBubble(pendingChat.message)
sendChat(pendingChat.message)
pendingChat = null
return 'SUCCESS'
})
])
```
### Complex Behavior Composition
The real power comes from combining nodes for complex behaviors:
```mermaid
graph TD
R[Root] --> S1[Chat Handler]
R --> S2[Zone Handler]
R --> S3[Idle Handler]
S1 --> C1[Has Chat?]
S1 --> C2[In Range?]
S1 --> A1[Process Chat]
S2 --> C3[Check Distance]
S2 --> C4[Zone Changed?]
S2 --> A2[Update Zone]
```
#### Example of interaction sequence
```javascript
const behaviorTree = new Selector('root', [
new Sequence('closeInteraction', [
// Only runs for nearby players
new ConditionNode('isVeryClose', context =>
context.playerDistance <= 2
),
// Handles multiple behaviors in sequence
new ActionNode('greet', context => {
if (currentZone !== 'close') {
currentZone = 'close'
data.emote = { timer: 0, type: 'bow' }
vrm.setEmote(config.emote1?.url)
showBubble("Perfect! Now we can talk.")
}
updateRotation(context)
return 'SUCCESS'
})
])
])
```
### Benefits over FSMs
1. Modularity
- Each node is self-contained
- Behaviors can be easily added/removed
- Common behaviors can be reused
2. Readability
- Tree structure visualizes logic flow
- Conditions and actions are clearly separated
- Behavior hierarchy is explicit
3. Maintainability
- Changes are localized to specific nodes
- Testing can focus on individual behaviors
- Complex behaviors are built from simple ones
4. Flexibility
- Multiple behaviors can run together
- Priority handling is built-in
- State transitions are implicit
### Code
```javascript
const BUBBLE_TIME = 5
const EMOTE_TIME = 2
const LOOK_TIME = 5
const UP = new Vector3(0, 1, 0)
const v1 = new Vector3()
const vrm = app.get('avatar')
// SERVER
if (world.isServer) {
const config = app.config
const state = { ready: true }
app.state = state
app.send('state', state)
const ctrl = app.create('controller')
ctrl.position.copy(app.position)
world.add(ctrl)
ctrl.quaternion.copy(app.quaternion)
ctrl.add(vrm)
const emoteUrls = {}
if (config.emote1Name && config.emote1?.url) {
emoteUrls[config.emote1Name] = config.emote1.url
}
if (config.emote2Name && config.emote2?.url) {
emoteUrls[config.emote2Name] = config.emote2.url
}
if (config.emote3Name && config.emote3?.url) {
emoteUrls[config.emote3Name] = config.emote3.url
}
if (config.emote4Name && config.emote4?.url) {
emoteUrls[config.emote4Name] = config.emote4.url
}
let changed = true
let notifying = false
const info = {
world: {
id: null,
name: null,
url: null,
context: config.context || 'You are in a virtual world powered by Hyperfy'
},
you: {
id: app.instanceId,
name: config.name,
},
emotes: Object.keys(emoteUrls),
triggers: [],
events: [],
}
world.on('enter', player => {
info.events.push({
type: 'player-enter',
playerId: player.id
})
changed = true
})
world.on('leave', player => {
info.events.push({
type: 'player-leave',
playerId: player.id
})
changed = true
})
world.on('chat', msg => {
if (msg.fromId === app.instanceId) return
if (msg.skipBehavior) return
const player = world.getPlayer(msg.fromId)
if (!player) return
// Add distance check before processing any chat events
const distance = player.position.distanceTo(ctrl.position)
if (distance <= 2) {
info.events.push({ type: 'chat', ...msg })
changed = true
}
})
async function notify() {
if (!config.url || notifying) return
changed = false
notifying = true
try {
const resp = await fetch(config.url, {
headers: { 'Content-Type': 'application/json' },
method: 'POST',
body: JSON.stringify(info)
})
const data = await resp.json()
if (data?.say) app.send('say', data.say)
if (data?.emote) app.send('emote', emoteUrls[data.emote])
if (data?.look) app.send('look', data.look)
} catch (err) {
console.error('notify failed:', err)
}
notifying = false
}
app.on('fixedUpdate', delta => {
if (changed && !notifying) notify()
})
}
// CLIENT
if (world.isClient) {
class BehaviorNode {
constructor(name) {
this.name = name
}
tick(context) { }
}
class Selector extends BehaviorNode {
constructor(name, children) {
super(name)
this.children = children
}
tick(context) {
for (const child of this.children) {
const status = child.tick(context)
if (status === 'SUCCESS' || status === 'RUNNING') {
return status
}
}
return 'FAILURE'
}
}
class Sequence extends BehaviorNode {
constructor(name, children) {
super(name)
this.children = children
}
tick(context) {
for (const child of this.children) {
const status = child.tick(context)
if (status !== 'SUCCESS') {
return status
}
}
return 'SUCCESS'
}
}
class ConditionNode extends BehaviorNode {
constructor(name, condition) {
super(name)
this.condition = condition
}
tick(context) {
return this.condition(context) ? 'SUCCESS' : 'FAILURE'
}
}
class ActionNode extends BehaviorNode {
constructor(name, action) {
super(name)
this.action = action
}
tick(context) {
return this.action(context)
}
}
const config = app.config
const idleEmoteUrl = config.emote0?.url
let currentZone = null
let messageCounter = 0
let pendingChat = null
const farMessages = [
"I can barely see you from here!",
"You're too far away to talk!",
"Come closer if you want to chat!",
"I can see you in the distance!"
]
const mediumMessages = [
"Almost there, just a bit closer!",
"I still can't quite hear you!",
"A few more steps closer please!",
"That's better, but come a little closer!"
]
function getNextMessage(messages) {
return messages[messageCounter++ % messages.length]
}
function sendChat(message, skipBehavior = false) {
world.chat({
id: uuid(),
from: config.name + ' (agent)',
fromId: app.instanceId,
body: message,
createdAt: world.getTimestamp(),
skipBehavior
}, true)
}
function showBubble(message) {
data.say = { timer: 0 }
bubble.active = true
nametag.active = false
bubbleText.value = message
}
function init() {
world.add(vrm)
vrm.setEmote(idleEmoteUrl)
}
// UI Setup
world.attach(vrm)
let state = app.state
if (state.ready) {
init()
} else {
world.remove(vrm)
app.on('state', _state => {
state = _state
init()
})
}
const bubble = app.create('ui')
bubble.width = 300
bubble.height = 512
bubble.size = 0.005
bubble.pivot = 'bottom-center'
bubble.billboard = 'full'
bubble.justifyContent = 'flex-end'
bubble.alignItems = 'center'
bubble.position.y = 2
bubble.active = false
const bubbleBox = app.create('uiview')
bubbleBox.backgroundColor = 'rgba(0, 0, 0, 0.95)'
bubbleBox.borderRadius = 20
bubbleBox.padding = 20
bubble.add(bubbleBox)
const bubbleText = app.create('uitext')
bubbleText.color = 'white'
bubbleText.fontWeight = 100
bubbleText.lineHeight = 1.4
bubbleText.fontSize = 16
bubbleText.value = '...'
bubbleBox.add(bubbleText)
vrm.add(bubble)
const nametag = app.create('nametag')
nametag.label = config.name
nametag.position.y = 2
vrm.add(nametag)
function updateRotation(context) {
if (context.player && context.playerDistance <= 10) {
const direction = v1.copy(context.player.position).sub(vrm.position)
direction.y = 0
const angle = Math.atan2(direction.x, direction.z) + Math.PI
vrm.quaternion.setFromAxisAngle(UP, angle)
}
}
const data = {}
const behaviorTree = new Selector('root', [
new Sequence('apiChatHandler', [
new ConditionNode('hasPendingApiChat', () =>
pendingChat && typeof pendingChat === 'object' && pendingChat.fromApi
),
new ActionNode('handleApiChat', () => {
showBubble(pendingChat.message)
sendChat(pendingChat.message, true)
pendingChat = null
return 'SUCCESS'
})
]),
new Sequence('playerChatHandler', [
new ConditionNode('hasPendingPlayerChat', () =>
pendingChat && typeof pendingChat === 'string'
),
new ActionNode('handlePlayerChat', context => {
if (context.playerDistance > 10) {
sendChat("I can't hear you from that far away! Come closer if you want to talk!", true)
} else if (context.playerDistance > 2) {
sendChat("If you want to speak to me, come closer!", true)
}
pendingChat = null
return 'SUCCESS'
})
]),
new Sequence('closeInteraction', [
new ConditionNode('isVeryClose', context => context.playerDistance <= 2),
new ActionNode('greet', context => {
if (currentZone !== 'close') {
currentZone = 'close'
data.emote = { timer: 0, type: 'bow' }
if (config.emote1?.url) {
vrm.setEmote(config.emote1.url)
}
showBubble("Ah, perfect! Now we can talk properly.")
sendChat("Ah, perfect! Now we can talk properly.", true)
}
updateRotation(context)
return 'SUCCESS'
})
]),
new Sequence('mediumInteraction', [
new ConditionNode('isMediumDistance', context =>
context.playerDistance > 2 && context.playerDistance <= 5
),
new ActionNode('mediumDistance', context => {
if (currentZone !== 'medium') {
currentZone = 'medium'
data.emote = { timer: 0, type: 'wave' }
if (config.emote2?.url) {
vrm.setEmote(config.emote2.url)
}
const message = getNextMessage(mediumMessages)
showBubble(message)
sendChat(message, true)
}
updateRotation(context)
return 'SUCCESS'
})
]),
new Sequence('farInteraction', [
new ConditionNode('isFarDistance', context =>
context.playerDistance > 5 && context.playerDistance <= 10
),
new ActionNode('farDistance', context => {
if (currentZone !== 'far') {
currentZone = 'far'
const message = getNextMessage(farMessages)
showBubble(message)
sendChat(message, true)
}
updateRotation(context)
return 'SUCCESS'
})
]),
new ActionNode('idle', context => {
if (currentZone !== 'idle') {
currentZone = 'idle'
if (!data.emote) {
vrm.setEmote(idleEmoteUrl)
}
bubble.active = false
nametag.active = true
}
return 'SUCCESS'
})
])
function updateContext(delta) {
const context = {
delta,
playerDistance: Infinity,
player: null
}
const player = world.getPlayer()
if (player) {
context.player = player
context.playerDistance = player.position.distanceTo(vrm.position)
}
return context
}
world.on('chat', msg => {
if (msg.fromId !== app.instanceId && !msg.skipBehavior) {
pendingChat = msg.body
}
})
app.on('say', value => {
pendingChat = { message: value, fromApi: true }
})
app.on('emote', url => {
if (currentZone !== 'idle') {
data.emote = { timer: 0 }
vrm.setEmote(url)
}
})
app.on('update', delta => {
const context = updateContext(delta)
behaviorTree.tick(context)
if (data.say?.timer !== undefined) {
data.say.timer += delta
if (data.say.timer > BUBBLE_TIME) {
data.say = null
bubble.active = false
nametag.active = true
}
}
if (data.emote?.timer !== undefined) {
data.emote.timer += delta
if (data.emote.timer > EMOTE_TIME) {
data.emote = null
vrm.setEmote(idleEmoteUrl)
}
}
})
}
// Configuration
app.configure(() => [
{ key: 'name', type: 'text', label: 'Name' },
{ key: 'context', type: 'textarea', label: 'Context' },
{ key: 'url', type: 'text', label: 'URL' },
{ key: 'emotes', type: 'section', label: 'Emotes' },
{ key: 'emote0', type: 'file', label: 'Idle', kind: 'emote' },
{
key: 'emote',
type: 'switch',
label: 'Custom',
options: [1, 2, 3, 4].map(n => ({ label: n.toString(), value: n.toString() }))
},
...[1, 2, 3, 4].flatMap(n => [
{
key: `emote${n}Name`,
type: 'text',
label: 'Name',
when: [{ key: 'emote', op: 'eq', value: n.toString() }]
},
{
key: `emote${n}`,
type: 'file',
label: 'Emote',
kind: 'emote',
when: [{ key: 'emote', op: 'eq', value: n.toString() }]
}
])
])
```
<!--
### Core Components and Flow
The behavior tree processes behaviors in a structured way. Each frame:
1. The context object is updated with new delta time
2. The root node "ticks" (processes) its children
3. Each node returns a status: SUCCESS, FAILURE, or RUNNING
4. Visual updates occur based on the executed actions
Here's how this flow works:
```mermaid
sequenceDiagram
participant C as Context
participant T as Tree
participant N as Nodes
participant V as Visual
C->>T: Update Delta Time
T->>N: Tick Root Node
N->>N: Process Children
N->>V: Execute Actions
V->>C: Update State
```
```javascript
const BUBBLE_TIME = 5
const EMOTE_TIME = 2
const LOOK_TIME = 5
const UP = new Vector3(0, 1, 0)
const v1 = new Vector3()
const v2 = new Vector3()
const v3 = new Vector3()
const q1 = new Quaternion()
const q2 = new Quaternion()
const m1 = new Matrix4()
const vrm = app.get('avatar')
// SERVER
if (world.isServer) {
const config = app.config
const state = {
ready: true,
}
app.state = state
app.send('state', state)
const ctrl = app.create('controller')
ctrl.position.copy(app.position)
world.add(ctrl)
ctrl.quaternion.copy(app.quaternion)
ctrl.add(vrm)
const emoteUrls = {}
if (config.emote1Name && config.emote1?.url) {
emoteUrls[config.emote1Name] = config.emote1.url
}
if (config.emote2Name && config.emote2?.url) {
emoteUrls[config.emote2Name] = config.emote2.url
}
if (config.emote3Name && config.emote3?.url) {
emoteUrls[config.emote3Name] = config.emote3.url
}
if (config.emote4Name && config.emote4?.url) {
emoteUrls[config.emote4Name] = config.emote4.url
}
// observe environment
let changed = true
let notifying = false
const info = {
world: {
id: null,
name: null,
url: null,
context: config.context || 'You are in a virtual world powered by Hyperfy'
},
you: {
id: app.instanceId,
name: config.name,
},
emotes: Object.keys(emoteUrls),
triggers: [],
events: [],
}
world.on('enter', player => {
info.events.push({
type: 'player-enter',
playerId: player.id
})
changed = true
})
world.on('leave', player => {
info.events.push({
type: 'player-leave',
playerId: player.id
})
changed = true
})
world.on('chat', msg => {
if (msg.fromId === app.instanceId) return
const player = world.getPlayer(msg.fromId)
if (!player) return
const distance = player.position.distanceTo(ctrl.position)
// If too far, just send the "come closer" message
if (distance > 2) {
const response = {
id: uuid(),
from: config.name + ' (agent)',
fromId: app.instanceId,
body: "If you want to speak to me, come closer!",
createdAt: world.getTimestamp()
}
world.chat(response, true)
return
}
// If in range, process with API
info.events.push({ type: 'chat', ...msg })
changed = true
})
async function notify() {
// Only notify if properly configured
if (!config.url || notifying) return
changed = false
notifying = true
console.log('notifying...', info)
let data
try {
const resp = await fetch(config.url, {
headers: {
'Content-Type': 'application/json',
},
method: 'POST',
body: JSON.stringify(info)
})
data = await resp.json()
} catch (err) {
console.error('notify failed')
}
notifying = false
if (!data) return
console.log(data)
if (data.say) {
app.send('say', data.say)
}
if (data.emote) {
const url = emoteUrls[data.emote]
app.send('emote', url)
}
if (data.look) {
app.send('look', data.look)
}
}
app.on('fixedUpdate', delta => {
if (changed && !notifying) {
notify()
}
})
}
// CLIENT
if (world.isClient) {
// ... Behavior Tree node classes stay the same ...
class BehaviorNode {
constructor(name) {
this.name = name
}
tick(context) { }
}
class Selector extends BehaviorNode {
constructor(name, children) {
super(name)
this.children = children
}
tick(context) {
for (const child of this.children) {
const status = child.tick(context)
if (status === 'SUCCESS' || status === 'RUNNING') {
return status
}
}
return 'FAILURE'
}
}
class Sequence extends BehaviorNode {
constructor(name, children) {
super(name)
this.children = children
}
tick(context) {
for (const child of this.children) {
const status = child.tick(context)
if (status !== 'SUCCESS') {
return status
}
}
return 'SUCCESS'
}
}
class ConditionNode extends BehaviorNode {
constructor(name, condition) {
super(name)
this.condition = condition
}
tick(context) {
return this.condition(context) ? 'SUCCESS' : 'FAILURE'
}
}
class ActionNode extends BehaviorNode {
constructor(name, action) {
super(name)
this.action = action
}
tick(context) {
return this.action(context)
}
}
const config = app.config
const idleEmoteUrl = config.emote0?.url
let currentZone = null
let messageCounter = 0
let pendingChat = null
const farMessages = [
"I can barely see you from here!",
"You're too far away to talk!",
"Come closer if you want to chat!",
"I can see you in the distance!"
]
const mediumMessages = [
"Almost there, just a bit closer!",
"I still can't quite hear you!",
"A few more steps closer please!",
"That's better, but come a little closer!"
]
const getNextMessage = (messages) => {
const msg = messages[messageCounter % messages.length]
messageCounter++
return msg
}
function sendChat(message) {
const msg = {
id: uuid(),
from: config.name + ' (agent)',
fromId: app.instanceId,
body: message,
createdAt: world.getTimestamp()
}
world.chat(msg, true)
}
world.attach(vrm)
let state = app.state
if (state.ready) {
init()
} else {
world.remove(vrm)
app.on('state', _state => {
state = _state
init()
})
}
// setup bubble and nametag (stays the same)
const bubble = app.create('ui')
bubble.width = 300
bubble.height = 512
bubble.size = 0.005
bubble.pivot = 'bottom-center'
bubble.billboard = 'full'
bubble.justifyContent = 'flex-end'
bubble.alignItems = 'center'
bubble.position.y = 2
bubble.active = false
const bubbleBox = app.create('uiview')
bubbleBox.backgroundColor = 'rgba(0, 0, 0, 0.95)'
bubbleBox.borderRadius = 20
bubbleBox.padding = 20
bubble.add(bubbleBox)
const bubbleText = app.create('uitext')
bubbleText.color = 'white'
bubbleText.fontWeight = 100
bubbleText.lineHeight = 1.4
bubbleText.fontSize = 16
bubbleText.value = '...'
bubbleBox.add(bubbleText)
vrm.add(bubble)
const nametag = app.create('nametag')
nametag.label = config.name
nametag.position.y = 2
vrm.add(nametag)
function init() {
world.add(vrm)
vrm.setEmote(idleEmoteUrl)
}
function updateRotation(context) {
if (context.player && context.playerDistance <= 10) {
const direction = v1.copy(context.player.position).sub(vrm.position)
direction.y = 0
const angle = Math.atan2(direction.x, direction.z) + Math.PI
vrm.quaternion.setFromAxisAngle(UP, angle)
}
}
function handleChatAtDistance(context) {
if (!pendingChat) return 'SUCCESS'
if (context.playerDistance > 10) {
sendChat("I can't hear you from that far away! Come closer if you want to talk.")
pendingChat = null
return 'SUCCESS'
}
if (context.playerDistance > 2) {
sendChat("If you want to speak to me, come closer!")
pendingChat = null
return 'SUCCESS'
}
// If we're in close range, handle the chat normally
data.say = { timer: 0 }
bubble.active = true
nametag.active = false
bubbleText.value = pendingChat
sendChat(pendingChat)
pendingChat = null
return 'SUCCESS'
}
const data = {}
const behaviorTree = new Selector('root', [
// Chat handling (checked first)
new Sequence('chatHandler', [
new ConditionNode('hasPendingChat', (context) => {
return pendingChat !== null
}),
new ActionNode('handleChat', (context) => {
return handleChatAtDistance(context)
})
]),
// Zone-based behaviors (same as before)
new Sequence('closeInteraction', [
new ConditionNode('isVeryClose', (context) => {
return context.playerDistance <= 2
}),
new ActionNode('greet', (context) => {
if (currentZone !== 'close') {
currentZone = 'close'
data.emote = { timer: 0, type: 'bow' }
if (config.emote1?.url) {
vrm.setEmote(config.emote1.url)
}
const message = "Ah, perfect! Now we can talk properly."
data.say = { timer: 0 }
bubbleText.value = message
bubble.active = true
nametag.active = false
sendChat(message)
}
updateRotation(context)
return 'SUCCESS'
})
]),
new Sequence('mediumInteraction', [
new ConditionNode('isMediumDistance', (context) => {
return context.playerDistance > 2 && context.playerDistance <= 5
}),
new ActionNode('mediumDistance', (context) => {
if (currentZone !== 'medium') {
currentZone = 'medium'
data.emote = { timer: 0, type: 'wave' }
if (config.emote2?.url) {
vrm.setEmote(config.emote2.url)
}
const message = getNextMessage(mediumMessages)
data.say = { timer: 0 }
bubbleText.value = message
bubble.active = true
nametag.active = false
sendChat(message)
}
updateRotation(context)
return 'SUCCESS'
})
]),
new Sequence('farInteraction', [
new ConditionNode('isFarDistance', (context) => {
return context.playerDistance > 5 && context.playerDistance <= 10
}),
new ActionNode('farDistance', (context) => {
if (currentZone !== 'far') {
currentZone = 'far'
const message = getNextMessage(farMessages)
data.say = { timer: 0 }
bubbleText.value = message
bubble.active = true
nametag.active = false
sendChat(message)
}
updateRotation(context)
return 'SUCCESS'
})
]),
new ActionNode('idle', (context) => {
if (currentZone !== 'idle') {
currentZone = 'idle'
if (!data.emote) {
vrm.setEmote(idleEmoteUrl)
}
bubble.active = false
nametag.active = true
}
return 'SUCCESS'
})
])
function updateContext(delta) {
const context = {
delta,
playerDistance: Infinity,
player: null
}
const player = world.getPlayer()
if (player) {
context.player = player
context.playerDistance = player.position.distanceTo(vrm.position)
}
return context
}
world.on('chat', msg => {
if (msg.fromId === app.instanceId) return
pendingChat = msg.body
})
app.on('say', value => {
// Queue the chat message for the behavior tree to handle
pendingChat = value
})
app.on('emote', url => {
// Allow emotes in all ranges except idle
if (currentZone !== 'idle') {
data.emote = { timer: 0 }
vrm.setEmote(url)
}
})
app.on('update', delta => {
const context = updateContext(delta)
behaviorTree.tick(context)
if (data.say) {
data.say.timer += delta
if (data.say.timer > BUBBLE_TIME) {
data.say = null
bubble.active = false
nametag.active = true
}
}
if (data.emote) {
data.emote.timer += delta
if (data.emote.timer > EMOTE_TIME) {
data.emote = null
vrm.setEmote(idleEmoteUrl)
}
}
})
}
// CONFIG stays exactly the same...
app.configure(() => {
return [
{
key: 'name',
type: 'text',
label: 'Name',
},
{
key: 'context',
type: 'textarea',
label: 'Context',
},
{
key: 'url',
type: 'text',
label: 'URL',
},
{
key: 'emotes',
type: 'section',
label: 'Emotes',
},
{
key: 'emote0',
type: 'file',
label: 'Idle',
kind: 'emote',
},
{
key: 'emote',
type: 'switch',
label: 'Custom',
options: [
{ label: '1', value: '1' },
{ label: '2', value: '2' },
{ label: '3', value: '3' },
{ label: '4', value: '4' },
],
},
...customEmoteFields('1'),
...customEmoteFields('2'),
...customEmoteFields('3'),
...customEmoteFields('4'),
];
function customEmoteFields(n) {
return [
{
key: `emote${n}Name`,
type: 'text',
label: 'Name',
when: [{ key: 'emote', op: 'eq', value: n }],
},
{
key: `emote${n}`,
type: 'file',
label: 'Emote',
kind: 'emote',
when: [{ key: 'emote', op: 'eq', value: n }],
},
];
}
}); -->
Sign in with Wallet
Connect another wallet