# RPC Submits Analysis
I've collected the backtraces of threads which call `socket_submit_outgoing_msg` when a client writes to a file. It is not clear if I've collected all calls, since the execution aborts at various points with the message `-bash: glusterfs-client/testfile: Transport endpoint is not connected`.
The backtraces are written below. The furthest I've gotten is 7 calls.
1. `iotwr` performs `lookup`
2. `iotwr` performs `ping`
3. `iotwr` performs `lookup`
4. `iotwr` performs `ping`
5. `epoll` performs `dump`
6. `epoll` performs `ping`
7. `iotwr` performs `ping`
My guess guess is that for each server, the `iowr`s perform a lookup and the `epoll`s dump the contents to be sent. I haven't managed to get a clean run to confirm this.
Continue:
- Get a "clean" run with all calls without exception
- Where do the dump calls come from?
- Investigate the `rpc_clnt_submit` function
- Especially the arguments `prog` and `procnum`
- Thread continuation after `submit` call
- Where is the response by the server handled?
## Trace1 IOT
```c
#0 socket_submit_outgoing_msg (this=0x61a000014a98, msg=0x7fffee9e77b0) at socket.c:3641
#1 0x00007ffff7218e4a in rpc_transport_submit_request (this=0x61a000014a98, req=req@entry=0x7fffee9e77b0)
at rpc-transport.c:388
#2 0x00007ffff7221db9 in rpc_clnt_submit (rpc=rpc@entry=0x614000010458, prog=prog@entry=0x7fffef099540 <clnt4_0_fop_prog>,
procnum=procnum@entry=27, cbkfn=cbkfn@entry=0x7fffef0091e0 <client4_0_lookup_cbk>, proghdr=proghdr@entry=0x7fffee9e7e00,
proghdrcount=proghdrcount@entry=1, progpayload=<optimized out>, progpayloadcount=<optimized out>,
iobref=<optimized out>, frame=<optimized out>, rsphdr=<optimized out>, rsphdr_count=<optimized out>,
rsp_payload=<optimized out>, rsp_payload_count=<optimized out>, rsp_iobref=<optimized out>) at rpc-clnt.c:1633
#3 0x00007fffeefe3bb7 in client_submit_request (this=this@entry=0x621000021d18, req=req@entry=0x7fffee9e8190,
frame=frame@entry=0x6110000ffcd8, prog=<optimized out>, procnum=procnum@entry=27,
cbkfn=cbkfn@entry=0x7fffef0091e0 <client4_0_lookup_cbk>, cp=<optimized out>, xdrproc=<optimized out>) at client.c:261
#4 0x00007fffef02b854 in client4_0_lookup (frame=frame@entry=0x6110000ffcd8, this=this@entry=0x621000021d18,
data=data@entry=0x7fffee9e8400) at client-rpc-fops_v2.c:2986
#5 0x00007fffeefca376 in client_lookup (frame=frame@entry=0x6110000ffcd8, this=this@entry=0x621000021d18,
loc=loc@entry=0x61d000310990, xdata=<optimized out>) at client.c:375
#6 0x00007fffeee779d6 in ec_wind_lookup (ec=0x61600001fe98, fop=0x61d000310698, idx=<optimized out>) at ec-generic.c:792
#7 0x00007fffeee66fc2 in ec_dispatch_mask (fop=0x61d000310698, mask=5) at ec-common.c:795
#8 0x00007fffeee676fa in ec_dispatch_all (fop=0x61d000310698) at ec-common.c:877
#9 0x00007fffeee7f84a in ec_manager_lookup (fop=<optimized out>, state=<optimized out>) at ec-generic.c:842
#10 0x00007fffeee63ed1 in __ec_manager (fop=0x61d000310698, error=<optimized out>) at ec-common.c:3017
#11 0x00007fffeee73905 in ec_manager (fop=<optimized out>, error=<optimized out>) at ec-common.c:3035
#12 0x00007fffeee80201 in ec_lookup (frame=0x6110000ff918, this=<optimized out>, target=<optimized out>,
fop_flags=<optimized out>, func=0x7ffff749a7e0 <default_lookup_cbk>, data=0x0, loc=0x6230001ff150, xdata=0x60d0000191b8)
at ec-generic.c:944
#13 0x00007fffeee51f74 in ec_gf_lookup (frame=frame@entry=0x6110000ff918, this=this@entry=0x621000025918,
loc=loc@entry=0x6230001ff150, xdata=xdata@entry=0x60d0000191b8) at ec.c:1164
#14 0x00007ffff7484070 in default_lookup (frame=frame@entry=0x6110000ff918, this=this@entry=0x621000026d18,
loc=loc@entry=0x6230001ff150, xdata=xdata@entry=0x60d0000191b8) at defaults.c:3024
#15 0x00007fffeebd28da in gf_utime_lookup (frame=frame@entry=0x6110000ff7d8, this=this@entry=0x621000028118,
loc=loc@entry=0x6230001ff150, xdata=xdata@entry=0x60d0000191b8) at ../../../../xlators/features/utime/src/utime.c:260
#16 0x00007fffeeb98231 in wb_lookup_helper (frame=<optimized out>, this=0x621000029518, loc=0x6230001ff150,
xdata=0x60d0000191b8) at write-behind.c:2379
#17 0x00007ffff731dcfe in call_resume_wind (stub=0x6230001ff118) at call-stub.c:1876
#18 0x00007ffff734df9d in call_resume_keep_stub (stub=0x6230001ff118) at call-stub.c:2455
#19 0x00007fffeeba8fad in wb_do_winds (wb_inode=<optimized out>, tasks=0x7fffee9e89f0) at write-behind.c:1722
#20 0x00007fffeeba9437 in wb_process_queue (wb_inode=wb_inode@entry=0x611000044b18) at write-behind.c:1759
#21 0x00007fffeebb2d45 in wb_lookup (frame=frame@entry=0x6110000ff698, this=this@entry=0x621000029518,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at write-behind.c:2401
#22 0x00007ffff7484070 in default_lookup (frame=frame@entry=0x6110000ff698, this=this@entry=0x62100002a918,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at defaults.c:3024
#23 0x00007fffeeb56b2d in qr_lookup (frame=frame@entry=0x6110000ff558, this=this@entry=0x62100002bd18,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at quick-read.c:647
#24 0x00007fffeeb319e3 in mdc_lookup (frame=frame@entry=0x6110000ff2d8, this=this@entry=0x62100002d118,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at md-cache.c:1337
#25 0x00007ffff74e6e51 in default_lookup_resume (frame=<optimized out>, this=0x62100002f918, loc=0x6230001f9d50,
xdata=0x60d0000191b8) at defaults.c:2215
#26 0x00007ffff731dcfe in call_resume_wind (stub=0x6230001f9d18) at call-stub.c:1876
#27 0x00007ffff734d8f5 in call_resume (stub=0x6230001f9d18) at call-stub.c:2390
#28 0x00007fffeead68bd in iot_worker (data=0x615000010498) at io-threads.c:227
#29 0x00007ffff71bf609 in start_thread (arg=<optimized out>) at pthread_create.c:477
#30 0x00007ffff70e4103 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
```
## Trace 2 IOT
```c
#0 socket_submit_outgoing_msg (this=0x61a000014a98, msg=0x7fffee9e6f30) at socket.c:3641
#1 0x00007ffff7218e4a in rpc_transport_submit_request (this=0x61a000014a98, req=req@entry=0x7fffee9e6f30)
at rpc-transport.c:388
#2 0x00007ffff7221db9 in rpc_clnt_submit (rpc=rpc@entry=0x614000010458, prog=prog@entry=0x7ffff72663c0 <clnt_ping_prog>,
procnum=procnum@entry=2, cbkfn=cbkfn@entry=0x7ffff722b730 <rpc_clnt_ping_cbk>, proghdr=proghdr@entry=0x0,
proghdrcount=proghdrcount@entry=0, progpayload=<optimized out>, progpayloadcount=<optimized out>,
iobref=<optimized out>, frame=<optimized out>, rsphdr=<optimized out>, rsphdr_count=<optimized out>,
rsp_payload=<optimized out>, rsp_payload_count=<optimized out>, rsp_iobref=<optimized out>) at rpc-clnt.c:1633
#3 0x00007ffff722c4ca in rpc_clnt_ping (rpc=rpc@entry=0x614000010458) at rpc-clnt-ping.c:261
#4 0x00007ffff722c9a8 in rpc_clnt_start_ping (rpc_ptr=0x614000010458) at rpc-clnt-ping.c:336
#5 rpc_clnt_start_ping (rpc_ptr=0x614000010458) at rpc-clnt-ping.c:279
#6 0x00007ffff722cb0d in rpc_clnt_check_and_start_ping (rpc=0x614000010458) at rpc-clnt-ping.c:352
#7 0x00007ffff7221e5c in rpc_clnt_submit (rpc=rpc@entry=0x614000010458, prog=prog@entry=0x7fffef099540 <clnt4_0_fop_prog>,
procnum=procnum@entry=27, cbkfn=cbkfn@entry=0x7fffef0091e0 <client4_0_lookup_cbk>, proghdr=proghdr@entry=0x7fffee9e7e00,
proghdrcount=proghdrcount@entry=1, progpayload=<optimized out>, progpayloadcount=<optimized out>,
iobref=<optimized out>, frame=<optimized out>, rsphdr=<optimized out>, rsphdr_count=<optimized out>,
rsp_payload=<optimized out>, rsp_payload_count=<optimized out>, rsp_iobref=<optimized out>) at rpc-clnt.c:1674
#8 0x00007fffeefe3bb7 in client_submit_request (this=this@entry=0x621000021d18, req=req@entry=0x7fffee9e8190,
frame=frame@entry=0x6110000ffcd8, prog=<optimized out>, procnum=procnum@entry=27,
cbkfn=cbkfn@entry=0x7fffef0091e0 <client4_0_lookup_cbk>, cp=<optimized out>, xdrproc=<optimized out>) at client.c:261
#9 0x00007fffef02b854 in client4_0_lookup (frame=frame@entry=0x6110000ffcd8, this=this@entry=0x621000021d18,
data=data@entry=0x7fffee9e8400) at client-rpc-fops_v2.c:2986
#10 0x00007fffeefca376 in client_lookup (frame=frame@entry=0x6110000ffcd8, this=this@entry=0x621000021d18,
loc=loc@entry=0x61d000310990, xdata=<optimized out>) at client.c:375
#11 0x00007fffeee779d6 in ec_wind_lookup (ec=0x61600001fe98, fop=0x61d000310698, idx=<optimized out>) at ec-generic.c:792
#12 0x00007fffeee66fc2 in ec_dispatch_mask (fop=0x61d000310698, mask=5) at ec-common.c:795
#13 0x00007fffeee676fa in ec_dispatch_all (fop=0x61d000310698) at ec-common.c:877
#14 0x00007fffeee7f84a in ec_manager_lookup (fop=<optimized out>, state=<optimized out>) at ec-generic.c:842
#15 0x00007fffeee63ed1 in __ec_manager (fop=0x61d000310698, error=<optimized out>) at ec-common.c:3017
#16 0x00007fffeee73905 in ec_manager (fop=<optimized out>, error=<optimized out>) at ec-common.c:3035
#17 0x00007fffeee80201 in ec_lookup (frame=0x6110000ff918, this=<optimized out>, target=<optimized out>,
fop_flags=<optimized out>, func=0x7ffff749a7e0 <default_lookup_cbk>, data=0x0, loc=0x6230001ff150, xdata=0x60d0000191b8)
at ec-generic.c:944
#18 0x00007fffeee51f74 in ec_gf_lookup (frame=frame@entry=0x6110000ff918, this=this@entry=0x621000025918,
loc=loc@entry=0x6230001ff150, xdata=xdata@entry=0x60d0000191b8) at ec.c:1164
#19 0x00007ffff7484070 in default_lookup (frame=frame@entry=0x6110000ff918, this=this@entry=0x621000026d18,
loc=loc@entry=0x6230001ff150, xdata=xdata@entry=0x60d0000191b8) at defaults.c:3024
#20 0x00007fffeebd28da in gf_utime_lookup (frame=frame@entry=0x6110000ff7d8, this=this@entry=0x621000028118,
loc=loc@entry=0x6230001ff150, xdata=xdata@entry=0x60d0000191b8) at ../../../../xlators/features/utime/src/utime.c:260
#21 0x00007fffeeb98231 in wb_lookup_helper (frame=<optimized out>, this=0x621000029518, loc=0x6230001ff150,
xdata=0x60d0000191b8) at write-behind.c:2379
#22 0x00007ffff731dcfe in call_resume_wind (stub=0x6230001ff118) at call-stub.c:1876
#23 0x00007ffff734df9d in call_resume_keep_stub (stub=0x6230001ff118) at call-stub.c:2455
#24 0x00007fffeeba8fad in wb_do_winds (wb_inode=<optimized out>, tasks=0x7fffee9e89f0) at write-behind.c:1722
#25 0x00007fffeeba9437 in wb_process_queue (wb_inode=wb_inode@entry=0x611000044b18) at write-behind.c:1759
#26 0x00007fffeebb2d45 in wb_lookup (frame=frame@entry=0x6110000ff698, this=this@entry=0x621000029518,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at write-behind.c:2401
#27 0x00007ffff7484070 in default_lookup (frame=frame@entry=0x6110000ff698, this=this@entry=0x62100002a918,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at defaults.c:3024
#28 0x00007fffeeb56b2d in qr_lookup (frame=frame@entry=0x6110000ff558, this=this@entry=0x62100002bd18,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at quick-read.c:647
#29 0x00007fffeeb319e3 in mdc_lookup (frame=frame@entry=0x6110000ff2d8, this=this@entry=0x62100002d118,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at defaults.c:3024
#28 0x00007fffeeb56b2d in qr_lookup (frame=frame@entry=0x6110000ff558, this=this@entry=0x62100002bd18,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at quick-read.c:647
#29 0x00007fffeeb319e3 in mdc_lookup (frame=frame@entry=0x6110000ff2d8, this=this@entry=0x62100002d118,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at md-cache.c:1337
#30 0x00007ffff74e6e51 in default_lookup_resume (frame=<optimized out>, this=0x62100002f918, loc=0x6230001f9d50,
xdata=0x60d0000191b8) at defaults.c:2215
#31 0x00007ffff731dcfe in call_resume_wind (stub=0x6230001f9d18) at call-stub.c:1876
#32 0x00007ffff734d8f5 in call_resume (stub=0x6230001f9d18) at call-stub.c:2390
#33 0x00007fffeead68bd in iot_worker (data=0x615000010498) at io-threads.c:227
#34 0x00007ffff71bf609 in start_thread (arg=<optimized out>) at pthread_create.c:477
#35 0x00007ffff70e4103 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
```
## Trace 3 IOT
```c
#0 socket_submit_outgoing_msg (this=0x61a000013e98, msg=0x7fffee9e77b0) at socket.c:3641
#1 0x00007ffff7218e4a in rpc_transport_submit_request (this=0x61a000013e98, req=req@entry=0x7fffee9e77b0)
at rpc-transport.c:388
#2 0x00007ffff7221db9 in rpc_clnt_submit (rpc=rpc@entry=0x614000010058, prog=prog@entry=0x7fffef099540 <clnt4_0_fop_prog>,
procnum=procnum@entry=27, cbkfn=cbkfn@entry=0x7fffef0091e0 <client4_0_lookup_cbk>, proghdr=proghdr@entry=0x7fffee9e7e00,
proghdrcount=proghdrcount@entry=1, progpayload=<optimized out>, progpayloadcount=<optimized out>,
iobref=<optimized out>, frame=<optimized out>, rsphdr=<optimized out>, rsphdr_count=<optimized out>,
rsp_payload=<optimized out>, rsp_payload_count=<optimized out>, rsp_iobref=<optimized out>) at rpc-clnt.c:1633
#3 0x00007fffeefe3bb7 in client_submit_request (this=this@entry=0x621000024518, req=req@entry=0x7fffee9e8190,
frame=frame@entry=0x61100015ff98, prog=<optimized out>, procnum=procnum@entry=27,
cbkfn=cbkfn@entry=0x7fffef0091e0 <client4_0_lookup_cbk>, cp=<optimized out>, xdrproc=<optimized out>) at client.c:261
#4 0x00007fffef02b854 in client4_0_lookup (frame=frame@entry=0x61100015ff98, this=this@entry=0x621000024518,
data=data@entry=0x7fffee9e8400) at client-rpc-fops_v2.c:2986
#5 0x00007fffeefca376 in client_lookup (frame=frame@entry=0x61100015ff98, this=this@entry=0x621000024518,
loc=loc@entry=0x61d000310990, xdata=<optimized out>) at client.c:375
#6 0x00007fffeee779d6 in ec_wind_lookup (ec=0x61600001fe98, fop=0x61d000310698, idx=<optimized out>) at ec-generic.c:792
#7 0x00007fffeee66fc2 in ec_dispatch_mask (fop=0x61d000310698, mask=1) at ec-common.c:795
#8 0x00007fffeee676fa in ec_dispatch_all (fop=0x61d000310698) at ec-common.c:877
#9 0x00007fffeee7f84a in ec_manager_lookup (fop=<optimized out>, state=<optimized out>) at ec-generic.c:842
#10 0x00007fffeee63ed1 in __ec_manager (fop=0x61d000310698, error=<optimized out>) at ec-common.c:3017
#11 0x00007fffeee73905 in ec_manager (fop=<optimized out>, error=<optimized out>) at ec-common.c:3035
#12 0x00007fffeee80201 in ec_lookup (frame=0x6110000ff918, this=<optimized out>, target=<optimized out>,
fop_flags=<optimized out>, func=0x7ffff749a7e0 <default_lookup_cbk>, data=0x0, loc=0x6230001ff150, xdata=0x60d0000191b8)
at ec-generic.c:944
#13 0x00007fffeee51f74 in ec_gf_lookup (frame=frame@entry=0x6110000ff918, this=this@entry=0x621000025918,
loc=loc@entry=0x6230001ff150, xdata=xdata@entry=0x60d0000191b8) at ec.c:1164
#14 0x00007ffff7484070 in default_lookup (frame=frame@entry=0x6110000ff918, this=this@entry=0x621000026d18,
loc=loc@entry=0x6230001ff150, xdata=xdata@entry=0x60d0000191b8) at defaults.c:3024
#15 0x00007fffeebd28da in gf_utime_lookup (frame=frame@entry=0x6110000ff7d8, this=this@entry=0x621000028118,
loc=loc@entry=0x6230001ff150, xdata=xdata@entry=0x60d0000191b8) at ../../../../xlators/features/utime/src/utime.c:260
#16 0x00007fffeeb98231 in wb_lookup_helper (frame=<optimized out>, this=0x621000029518, loc=0x6230001ff150,
xdata=0x60d0000191b8) at write-behind.c:2379
#17 0x00007ffff731dcfe in call_resume_wind (stub=0x6230001ff118) at call-stub.c:1876
#18 0x00007ffff734df9d in call_resume_keep_stub (stub=0x6230001ff118) at call-stub.c:2455
#19 0x00007fffeeba8fad in wb_do_winds (wb_inode=<optimized out>, tasks=0x7fffee9e89f0) at write-behind.c:1722
#20 0x00007fffeeba9437 in wb_process_queue (wb_inode=wb_inode@entry=0x611000044b18) at write-behind.c:1759
#21 0x00007fffeebb2d45 in wb_lookup (frame=frame@entry=0x6110000ff698, this=this@entry=0x621000029518,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at write-behind.c:2401
#22 0x00007ffff7484070 in default_lookup (frame=frame@entry=0x6110000ff698, this=this@entry=0x62100002a918,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at defaults.c:3024
#23 0x00007fffeeb56b2d in qr_lookup (frame=frame@entry=0x6110000ff558, this=this@entry=0x62100002bd18,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at quick-read.c:647
#24 0x00007fffeeb319e3 in mdc_lookup (frame=frame@entry=0x6110000ff2d8, this=this@entry=0x62100002d118,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at md-cache.c:1337
#25 0x00007ffff74e6e51 in default_lookup_resume (frame=<optimized out>, this=0x62100002f918, loc=0x6230001f9d50,
xdata=0x60d0000191b8) at defaults.c:2215
#26 0x00007ffff731dcfe in call_resume_wind (stub=0x6230001f9d18) at call-stub.c:1876
#27 0x00007ffff734d8f5 in call_resume (stub=0x6230001f9d18) at call-stub.c:2390
#28 0x00007fffeead68bd in iot_worker (data=0x615000010498) at io-threads.c:227
#29 0x00007ffff71bf609 in start_thread (arg=<optimized out>) at pthread_create.c:477
#30 0x00007ffff70e4103 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:9
```
## Trace 4 IOT
```c
#0 socket_submit_outgoing_msg (this=0x61a000013e98, msg=0x7fffee9e6f30) at socket.c:3641
#1 0x00007ffff7218e4a in rpc_transport_submit_request (this=0x61a000013e98, req=req@entry=0x7fffee9e6f30)
at rpc-transport.c:388
#2 0x00007ffff7221db9 in rpc_clnt_submit (rpc=rpc@entry=0x614000010058, prog=prog@entry=0x7ffff72663c0 <clnt_ping_prog>,
procnum=procnum@entry=2, cbkfn=cbkfn@entry=0x7ffff722b730 <rpc_clnt_ping_cbk>, proghdr=proghdr@entry=0x0,
proghdrcount=proghdrcount@entry=0, progpayload=<optimized out>, progpayloadcount=<optimized out>,
iobref=<optimized out>, frame=<optimized out>, rsphdr=<optimized out>, rsphdr_count=<optimized out>,
rsp_payload=<optimized out>, rsp_payload_count=<optimized out>, rsp_iobref=<optimized out>) at rpc-clnt.c:1633
#3 0x00007ffff722c4ca in rpc_clnt_ping (rpc=rpc@entry=0x614000010058) at rpc-clnt-ping.c:261
#4 0x00007ffff722c9a8 in rpc_clnt_start_ping (rpc_ptr=0x614000010058) at rpc-clnt-ping.c:336
#5 rpc_clnt_start_ping (rpc_ptr=0x614000010058) at rpc-clnt-ping.c:279
#6 0x00007ffff722cb0d in rpc_clnt_check_and_start_ping (rpc=0x614000010058) at rpc-clnt-ping.c:352
#7 0x00007ffff7221e5c in rpc_clnt_submit (rpc=rpc@entry=0x614000010058, prog=prog@entry=0x7fffef099540 <clnt4_0_fop_prog>,
procnum=procnum@entry=27, cbkfn=cbkfn@entry=0x7fffef0091e0 <client4_0_lookup_cbk>, proghdr=proghdr@entry=0x7fffee9e7e00,
proghdrcount=proghdrcount@entry=1, progpayload=<optimized out>, progpayloadcount=<optimized out>,
iobref=<optimized out>, frame=<optimized out>, rsphdr=<optimized out>, rsphdr_count=<optimized out>,
rsp_payload=<optimized out>, rsp_payload_count=<optimized out>, rsp_iobref=<optimized out>) at rpc-clnt.c:1674
#8 0x00007fffeefe3bb7 in client_submit_request (this=this@entry=0x621000024518, req=req@entry=0x7fffee9e8190,
frame=frame@entry=0x61100015ff98, prog=<optimized out>, procnum=procnum@entry=27,
cbkfn=cbkfn@entry=0x7fffef0091e0 <client4_0_lookup_cbk>, cp=<optimized out>, xdrproc=<optimized out>) at client.c:261
#9 0x00007fffef02b854 in client4_0_lookup (frame=frame@entry=0x61100015ff98, this=this@entry=0x621000024518,
data=data@entry=0x7fffee9e8400) at client-rpc-fops_v2.c:2986
#10 0x00007fffeefca376 in client_lookup (frame=frame@entry=0x61100015ff98, this=this@entry=0x621000024518,
loc=loc@entry=0x61d000310990, xdata=<optimized out>) at client.c:375
#11 0x00007fffeee779d6 in ec_wind_lookup (ec=0x61600001fe98, fop=0x61d000310698, idx=<optimized out>) at ec-generic.c:792
#12 0x00007fffeee66fc2 in ec_dispatch_mask (fop=0x61d000310698, mask=1) at ec-common.c:795
#13 0x00007fffeee676fa in ec_dispatch_all (fop=0x61d000310698) at ec-common.c:877
#14 0x00007fffeee7f84a in ec_manager_lookup (fop=<optimized out>, state=<optimized out>) at ec-generic.c:842
#15 0x00007fffeee63ed1 in __ec_manager (fop=0x61d000310698, error=<optimized out>) at ec-common.c:3017
#16 0x00007fffeee73905 in ec_manager (fop=<optimized out>, error=<optimized out>) at ec-common.c:3035
#17 0x00007fffeee80201 in ec_lookup (frame=0x6110000ff918, this=<optimized out>, target=<optimized out>,
fop_flags=<optimized out>, func=0x7ffff749a7e0 <default_lookup_cbk>, data=0x0, loc=0x6230001ff150, xdata=0x60d0000191b8)
at ec-generic.c:944
#18 0x00007fffeee51f74 in ec_gf_lookup (frame=frame@entry=0x6110000ff918, this=this@entry=0x621000025918,
loc=loc@entry=0x6230001ff150, xdata=xdata@entry=0x60d0000191b8) at ec.c:1164
#19 0x00007ffff7484070 in default_lookup (frame=frame@entry=0x6110000ff918, this=this@entry=0x621000026d18,
loc=loc@entry=0x6230001ff150, xdata=xdata@entry=0x60d0000191b8) at defaults.c:3024
#20 0x00007fffeebd28da in gf_utime_lookup (frame=frame@entry=0x6110000ff7d8, this=this@entry=0x621000028118,
loc=loc@entry=0x6230001ff150, xdata=xdata@entry=0x60d0000191b8) at ../../../../xlators/features/utime/src/utime.c:260
#21 0x00007fffeeb98231 in wb_lookup_helper (frame=<optimized out>, this=0x621000029518, loc=0x6230001ff150,
xdata=0x60d0000191b8) at write-behind.c:2379
#22 0x00007ffff731dcfe in call_resume_wind (stub=0x6230001ff118) at call-stub.c:1876
#23 0x00007ffff734df9d in call_resume_keep_stub (stub=0x6230001ff118) at call-stub.c:2455
#24 0x00007fffeeba8fad in wb_do_winds (wb_inode=<optimized out>, tasks=0x7fffee9e89f0) at write-behind.c:1722
#25 0x00007fffeeba9437 in wb_process_queue (wb_inode=wb_inode@entry=0x611000044b18) at write-behind.c:1759
#26 0x00007fffeebb2d45 in wb_lookup (frame=frame@entry=0x6110000ff698, this=this@entry=0x621000029518,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at write-behind.c:2401
#27 0x00007ffff7484070 in default_lookup (frame=frame@entry=0x6110000ff698, this=this@entry=0x62100002a918,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at defaults.c:3024
#28 0x00007fffeeb56b2d in qr_lookup (frame=frame@entry=0x6110000ff558, this=this@entry=0x62100002bd18,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at quick-read.c:647
#29 0x00007fffeeb319e3 in mdc_lookup (frame=frame@entry=0x6110000ff2d8, this=this@entry=0x62100002d118,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at defaults.c:3024
#28 0x00007fffeeb56b2d in qr_lookup (frame=frame@entry=0x6110000ff558, this=this@entry=0x62100002bd18,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at quick-read.c:647
#29 0x00007fffeeb319e3 in mdc_lookup (frame=frame@entry=0x6110000ff2d8, this=this@entry=0x62100002d118,
loc=loc@entry=0x6230001f9d50, xdata=xdata@entry=0x60d0000191b8) at md-cache.c:1337
#30 0x00007ffff74e6e51 in default_lookup_resume (frame=<optimized out>, this=0x62100002f918, loc=0x6230001f9d50,
xdata=0x60d0000191b8) at defaults.c:2215
#31 0x00007ffff731dcfe in call_resume_wind (stub=0x6230001f9d18) at call-stub.c:1876
#32 0x00007ffff734d8f5 in call_resume (stub=0x6230001f9d18) at call-stub.c:2390
#33 0x00007fffeead68bd in iot_worker (data=0x615000010498) at io-threads.c:227
#34 0x00007ffff71bf609 in start_thread (arg=<optimized out>) at pthread_create.c:477
#35 0x00007ffff70e4103 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
```
## Trace 5 EPOLL T8
```c
#0 socket_submit_outgoing_msg (this=0x61a000014a98, msg=0x7fffef8b7c70) at socket.c:3641
#1 0x00007ffff7218e4a in rpc_transport_submit_request (this=0x61a000014a98, req=req@entry=0x7fffef8b7c70)
at rpc-transport.c:388
#2 0x00007ffff7221db9 in rpc_clnt_submit (rpc=rpc@entry=0x614000010458, prog=prog@entry=0x7fffef09da20 <clnt_dump_prog>,
procnum=procnum@entry=1, cbkfn=cbkfn@entry=0x7fffef057320 <client_dump_version_cbk>,
proghdr=proghdr@entry=0x7fffef8b82c0, proghdrcount=proghdrcount@entry=1, progpayload=<optimized out>,
progpayloadcount=<optimized out>, iobref=<optimized out>, frame=<optimized out>, rsphdr=<optimized out>,
rsphdr_count=<optimized out>, rsp_payload=<optimized out>, rsp_payload_count=<optimized out>, rsp_iobref=<optimized out>)
at rpc-clnt.c:1633
#3 0x00007fffeefe3bb7 in client_submit_request (this=this@entry=0x621000021d18, req=req@entry=0x7fffef8b85c0,
frame=frame@entry=0x6110001763d8, prog=<optimized out>, procnum=procnum@entry=1,
cbkfn=cbkfn@entry=0x7fffef057320 <client_dump_version_cbk>, cp=<optimized out>, xdrproc=<optimized out>) at client.c:261
#4 0x00007fffef0586c9 in client_handshake (this=this@entry=0x621000021d18, rpc=rpc@entry=0x614000010458)
at client-handshake.c:1117
#5 0x00007fffeefe2781 in client_rpc_notify (rpc=rpc@entry=0x614000010458, mydata=0x621000021d18,
event=event@entry=RPC_CLNT_CONNECT, data=data@entry=0x0) at client.c:2228
#6 0x00007ffff721f88d in rpc_clnt_notify (trans=trans@entry=0x61a000014a98, mydata=0x614000010488,
event=event@entry=RPC_TRANSPORT_CONNECT, data=data@entry=0x61a000014a98) at rpc-clnt.c:927
#7 0x00007ffff7219984 in rpc_transport_notify (this=this@entry=0x61a000014a98, event=event@entry=RPC_TRANSPORT_CONNECT,
data=data@entry=0x61a000014a98) at rpc-transport.c:521
#8 0x00007ffff01911fb in socket_connect_finish (this=this@entry=0x61a000014a98) at socket.c:2465
#9 0x00007ffff019a124 in socket_handle_client_connection_attempt (this=0x61a000014a98) at socket.c:2653
#10 socket_complete_connection (this=0x61a000014a98) at socket.c:2702
#11 socket_event_handler (event_thread_died=0, poll_err=<optimized out>, poll_out=4, poll_in=0, data=0x61a000014a98,
gen=145, idx=1, fd=7) at socket.c:2759
#12 socket_event_handler (fd=fd@entry=7, idx=idx@entry=1, gen=gen@entry=145, data=data@entry=0x61a000014a98,
poll_in=<optimized out>, poll_out=4, poll_err=<optimized out>, event_thread_died=<optimized out>) at socket.c:2710
#13 0x00007ffff73fa6c1 in event_dispatch_epoll_handler (event=0x7fffef8b8fe0, event_pool=0x629000000218) at event-epoll.c:631
#14 event_dispatch_epoll_worker (data=<optimized out>) at event-epoll.c:742
#15 0x00007ffff71bf609 in start_thread (arg=<optimized out>) at pthread_create.c:477
#16 0x00007ffff70e4103 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
```
## Trace 6 EPOLL T8
```c
#0 socket_submit_outgoing_msg (this=0x61a000014a98, msg=0x7fffef8b73f0) at socket.c:3641
#1 0x00007ffff7218e4a in rpc_transport_submit_request (this=0x61a000014a98, req=req@entry=0x7fffef8b73f0)
at rpc-transport.c:388
#2 0x00007ffff7221db9 in rpc_clnt_submit (rpc=rpc@entry=0x614000010458, prog=prog@entry=0x7ffff72663c0 <clnt_ping_prog>,
procnum=procnum@entry=2, cbkfn=cbkfn@entry=0x7ffff722b730 <rpc_clnt_ping_cbk>, proghdr=proghdr@entry=0x0,
proghdrcount=proghdrcount@entry=0, progpayload=<optimized out>, progpayloadcount=<optimized out>,
iobref=<optimized out>, frame=<optimized out>, rsphdr=<optimized out>, rsphdr_count=<optimized out>,
rsp_payload=<optimized out>, rsp_payload_count=<optimized out>, rsp_iobref=<optimized out>) at rpc-clnt.c:1633
#3 0x00007ffff722c4ca in rpc_clnt_ping (rpc=rpc@entry=0x614000010458) at rpc-clnt-ping.c:261
#4 0x00007ffff722c9a8 in rpc_clnt_start_ping (rpc_ptr=0x614000010458) at rpc-clnt-ping.c:336
#5 rpc_clnt_start_ping (rpc_ptr=0x614000010458) at rpc-clnt-ping.c:279
#6 0x00007ffff722cb0d in rpc_clnt_check_and_start_ping (rpc=0x614000010458) at rpc-clnt-ping.c:352
#7 0x00007ffff7221e5c in rpc_clnt_submit (rpc=rpc@entry=0x614000010458, prog=prog@entry=0x7fffef09da20 <clnt_dump_prog>,
procnum=procnum@entry=1, cbkfn=cbkfn@entry=0x7fffef057320 <client_dump_version_cbk>,
proghdr=proghdr@entry=0x7fffef8b82c0, proghdrcount=proghdrcount@entry=1, progpayload=<optimized out>,
progpayloadcount=<optimized out>, iobref=<optimized out>, frame=<optimized out>, rsphdr=<optimized out>,
rsphdr_count=<optimized out>, rsp_payload=<optimized out>, rsp_payload_count=<optimized out>, rsp_iobref=<optimized out>)
at rpc-clnt.c:1674
#8 0x00007fffeefe3bb7 in client_submit_request (this=this@entry=0x621000021d18, req=req@entry=0x7fffef8b85c0,
frame=frame@entry=0x6110001763d8, prog=<optimized out>, procnum=procnum@entry=1,
cbkfn=cbkfn@entry=0x7fffef057320 <client_dump_version_cbk>, cp=<optimized out>, xdrproc=<optimized out>) at client.c:261
#9 0x00007fffef0586c9 in client_handshake (this=this@entry=0x621000021d18, rpc=rpc@entry=0x614000010458)
at client-handshake.c:1117
#10 0x00007fffeefe2781 in client_rpc_notify (rpc=rpc@entry=0x614000010458, mydata=0x621000021d18,
event=event@entry=RPC_CLNT_CONNECT, data=data@entry=0x0) at client.c:2228
#11 0x00007ffff721f88d in rpc_clnt_notify (trans=trans@entry=0x61a000014a98, mydata=0x614000010488,
event=event@entry=RPC_TRANSPORT_CONNECT, data=data@entry=0x61a000014a98) at rpc-clnt.c:927
#12 0x00007ffff7219984 in rpc_transport_notify (this=this@entry=0x61a000014a98, event=event@entry=RPC_TRANSPORT_CONNECT,
data=data@entry=0x61a000014a98) at rpc-transport.c:521
#13 0x00007ffff01911fb in socket_connect_finish (this=this@entry=0x61a000014a98) at socket.c:2465
#14 0x00007ffff019a124 in socket_handle_client_connection_attempt (this=0x61a000014a98) at socket.c:2653
#15 socket_complete_connection (this=0x61a000014a98) at socket.c:2702
#16 socket_event_handler (event_thread_died=0, poll_err=<optimized out>, poll_out=4, poll_in=0, data=0x61a000014a98,
gen=145, idx=1, fd=7) at socket.c:2759
#17 socket_event_handler (fd=fd@entry=7, idx=idx@entry=1, gen=gen@entry=145, data=data@entry=0x61a000014a98,
poll_in=<optimized out>, poll_out=4, poll_err=<optimized out>, event_thread_died=<optimized out>) at socket.c:2710
#18 0x00007ffff73fa6c1 in event_dispatch_epoll_handler (event=0x7fffef8b8fe0, event_pool=0x629000000218) at event-epoll.c:631
#19 event_dispatch_epoll_worker (data=<optimized out>) at event-epoll.c:742
#20 0x00007ffff71bf609 in start_thread (arg=<optimized out>) at pthread_create.c:477
#21 0x00007ffff70e4103 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
```
## Trace 7 IOT
```c
#0 socket_submit_outgoing_msg (this=0x61a000013e98, msg=0x7fffee9e6f30) at socket.c:3641
#1 0x00007ffff7218e4a in rpc_transport_submit_request (this=0x61a000013e98, req=req@entry=0x7fffee9e6f30)
at rpc-transport.c:388
#2 0x00007ffff7221db9 in rpc_clnt_submit (rpc=rpc@entry=0x614000010058, prog=prog@entry=0x7ffff72663c0 <clnt_ping_prog>,
procnum=procnum@entry=2, cbkfn=cbkfn@entry=0x7ffff722b730 <rpc_clnt_ping_cbk>, proghdr=proghdr@entry=0x0,
proghdrcount=proghdrcount@entry=0, progpayload=<optimized out>, progpayloadcount=<optimized out>,
iobref=<optimized out>, frame=<optimized out>, rsphdr=<optimized out>, rsphdr_count=<optimized out>,
rsp_payload=<optimized out>, rsp_payload_count=<optimized out>, rsp_iobref=<optimized out>) at rpc-clnt.c:1633
#3 0x00007ffff722c4ca in rpc_clnt_ping (rpc=rpc@entry=0x614000010058) at rpc-clnt-ping.c:261
#4 0x00007ffff722c9a8 in rpc_clnt_start_ping (rpc_ptr=0x614000010058) at rpc-clnt-ping.c:336
#5 rpc_clnt_start_ping (rpc_ptr=0x614000010058) at rpc-clnt-ping.c:279
#6 0x00007ffff722cb0d in rpc_clnt_check_and_start_ping (rpc=0x614000010058) at rpc-clnt-ping.c:352
#7 0x00007ffff7221e5c in rpc_clnt_submit (rpc=rpc@entry=0x614000010058, prog=prog@entry=0x7fffef099540 <clnt4_0_fop_prog>,
procnum=procnum@entry=27, cbkfn=cbkfn@entry=0x7fffef0091e0 <client4_0_lookup_cbk>, proghdr=proghdr@entry=0x7fffee9e7e00,
proghdrcount=proghdrcount@entry=1, progpayload=<optimized out>, progpayloadcount=<optimized out>,
iobref=<optimized out>, frame=<optimized out>, rsphdr=<optimized out>, rsphdr_count=<optimized out>,
rsp_payload=<optimized out>, rsp_payload_count=<optimized out>, rsp_iobref=<optimized out>) at rpc-clnt.c:1674
#8 0x00007fffeefe3bb7 in client_submit_request (this=this@entry=0x621000024518, req=req@entry=0x7fffee9e8190,
frame=frame@entry=0x61100018b558, prog=<optimized out>, procnum=procnum@entry=27,
cbkfn=cbkfn@entry=0x7fffef0091e0 <client4_0_lookup_cbk>, cp=<optimized out>, xdrproc=<optimized out>) at client.c:261
#9 0x00007fffef02b854 in client4_0_lookup (frame=frame@entry=0x61100018b558, this=this@entry=0x621000024518,
data=data@entry=0x7fffee9e8400) at client-rpc-fops_v2.c:2986
#10 0x00007fffeefca376 in client_lookup (frame=frame@entry=0x61100018b558, this=this@entry=0x621000024518,
loc=loc@entry=0x61d00031b390, xdata=<optimized out>) at client.c:375
#11 0x00007fffeee779d6 in ec_wind_lookup (ec=0x61600001fe98, fop=0x61d00031b098, idx=<optimized out>) at ec-generic.c:792
#12 0x00007fffeee66fc2 in ec_dispatch_mask (fop=0x61d00031b098, mask=1) at ec-common.c:795
#13 0x00007fffeee676fa in ec_dispatch_all (fop=0x61d00031b098) at ec-common.c:877
#14 0x00007fffeee7f84a in ec_manager_lookup (fop=<optimized out>, state=<optimized out>) at ec-generic.c:842
#15 0x00007fffeee63ed1 in __ec_manager (fop=0x61d00031b098, error=<optimized out>) at ec-common.c:3017
#16 0x00007fffeee73905 in ec_manager (fop=<optimized out>, error=<optimized out>) at ec-common.c:3035
#17 0x00007fffeee80201 in ec_lookup (frame=0x61100018add8, this=<optimized out>, target=<optimized out>,
fop_flags=<optimized out>, func=0x7ffff749a7e0 <default_lookup_cbk>, data=0x0, loc=0x62300022ad50, xdata=0x60d000019c48)
at ec-generic.c:944
#18 0x00007fffeee51f74 in ec_gf_lookup (frame=frame@entry=0x61100018add8, this=this@entry=0x621000025918,
loc=loc@entry=0x62300022ad50, xdata=xdata@entry=0x60d000019c48) at ec.c:1164
#19 0x00007ffff7484070 in default_lookup (frame=frame@entry=0x61100018add8, this=this@entry=0x621000026d18,
loc=loc@entry=0x62300022ad50, xdata=xdata@entry=0x60d000019c48) at defaults.c:3024
#20 0x00007fffeebd28da in gf_utime_lookup (frame=frame@entry=0x61100018ac98, this=this@entry=0x621000028118,
loc=loc@entry=0x62300022ad50, xdata=xdata@entry=0x60d000019c48) at ../../../../xlators/features/utime/src/utime.c:260
#21 0x00007fffeeb98231 in wb_lookup_helper (frame=<optimized out>, this=0x621000029518, loc=0x62300022ad50,
xdata=0x60d000019c48) at write-behind.c:2379
#22 0x00007ffff731dcfe in call_resume_wind (stub=0x62300022ad18) at call-stub.c:1876
#23 0x00007ffff734df9d in call_resume_keep_stub (stub=0x62300022ad18) at call-stub.c:2455
#24 0x00007fffeeba8fad in wb_do_winds (wb_inode=<optimized out>, tasks=0x7fffee9e89f0) at write-behind.c:1722
#25 0x00007fffeeba9437 in wb_process_queue (wb_inode=wb_inode@entry=0x611000044b18) at write-behind.c:1759
#26 0x00007fffeebb2d45 in wb_lookup (frame=frame@entry=0x61100018ab58, this=this@entry=0x621000029518,
loc=loc@entry=0x623000235550, xdata=xdata@entry=0x60d000019c48) at write-behind.c:2401
#27 0x00007ffff7484070 in default_lookup (frame=frame@entry=0x61100018ab58, this=this@entry=0x62100002a918,
loc=loc@entry=0x623000235550, xdata=xdata@entry=0x60d000019c48) at defaults.c:3024
#28 0x00007fffeeb56b2d in qr_lookup (frame=frame@entry=0x61100018aa18, this=this@entry=0x62100002bd18,
loc=loc@entry=0x623000235550, xdata=xdata@entry=0x60d000019c48) at quick-read.c:647
#29 0x00007fffeeb319e3 in mdc_lookup (frame=frame@entry=0x61100018a798, this=this@entry=0x62100002d118,
loc=loc@entry=0x623000235550, xdata=xdata@entry=0x60d000019c48) at md-cache.c:1337
#30 0x00007ffff74e6e51 in default_lookup_resume (frame=<optimized out>, this=0x62100002f918, loc=0x623000235550,
xdata=0x60d000019c48) at defaults.c:2215
#31 0x00007ffff731dcfe in call_resume_wind (stub=0x623000235518) at call-stub.c:1876
#32 0x00007ffff734d8f5 in call_resume (stub=0x623000235518) at call-stub.c:2390
#33 0x00007fffeead68bd in iot_worker (data=0x615000010498) at io-threads.c:227
#34 0x00007ffff71bf609 in start_thread (arg=<optimized out>) at pthread_create.c:477
#35 0x00007ffff70e4103 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
```
Sign in with Wallet
Connect another wallet