---
tags: mPOS, troubleshoot, diagnose
---

# 20210616 observe Store228 memory leak [SOLVED]

## Cause and solution
Improper resource management. **`Jsonb`** needs to be closed after every initialization.

## Observation
### jmap output


### tomcat admin console

### jmap total MB (remote)

#### Before restart tomcat.service

#### After restart tomcat.service

#### After deploy new WAR & restart tomcat.service

### catalina log java memory exception

16-Jun-2021 09:45:30.330 INFO [http-nio-8080-exec-9] tk3c.mpos.mailman.GeneralFilter.log 1.API Starting SCProducts.get_ProductInfo [API] (Client: 0:0:0:0:0:0:0:1) GET http://localhost:9000/mPOS/product/123 | pluNo=123
16-Jun-2021 09:45:30.334 INFO [http-nio-8080-exec-9] tk3c.mpos.mailman.SCProducts.getProductJson getting product json...
16-Jun-2021 09:45:30.335 INFO [http-nio-8080-exec-9] tk3c.mpos.mailman.SCDataService.getConnection 2....getting connection from org.apache.tomcat.dbcp.dbcp2.BasicDataSource@1bb1f09 [DB]
16-Jun-2021 09:45:30.366 INFO [http-nio-8080-exec-9] tk3c.mpos.mailman.SCDataService.getConnection datasource connected [DB] -> Conn # 11922515 OPEN
  DATABASE:Oracle Database 11g Express Edition Release 11.2.0.2.0 - Production;DRIVER:11.2.0.4.0
16-Jun-2021 09:45:30.366 INFO [http-nio-8080-exec-9] tk3c.mpos.mailman.SCDataService.executeQuery WITH pluParam AS (SELECT plu_no, ? withMM FROM ( SELECT plu_no FROM product WHERE plu_no=? UNION ALL ( SELECT plu_no FROM product WHERE barcode1 = Lpad(?, 13, '0') UNION SELECT plu_no FROM product WHERE barcode2 = Lpad(?, 13, '0')) ) WHERE ROWNUM = 1), warr AS ( SELECT warr_accident.plu_no, Listagg(warr_kind||','||plu_name||','||warr_fee||','|| warr_year||','||plu_spec, ';' ) within GROUP(ORDER BY warr_kind) "warrantyInfo" FROM warr_accident JOIN product on product.plu_no=warr_accident.warr_kind WHERE begdate <= Trunc(SYSDATE) AND enddate >= Trunc(SYSDATE) GROUP BY warr_accident.plu_no ORDER BY "warrantyInfo"), pluPromo AS ( SELECT product.PLU_NO "pluNo", product.PLU_NAME "pluName", product.PLU_SPEC "pluSpec", product.PRICE "price", product.VIP_PRICE "vipPrice", product.BARCODE1 "barcode1", product.BARCODE2 "barcode2", product.FB_AMOUNT "fbAmount", product.SAL_TYPE "salType", product.DIRECT "direct", product.VALID_YEAR "validYear", product.NO_DUP_ITEM "noDupItem", product.ATTRIB "attrib", product.UNIT "measureWord", product.MIN_PRICE "minPrice" , CASE  WHEN attrib='U' THEN 'WORDING='||product.plu_spec ELSE (SELECT keyvalue from para_store where keyvalue LIKE '#PRODNO=' || product.plu_no || '%' AND rownum=1) END "paraVal", product.auth_model "authModel", (select "warrantyInfo" FROM warr WHERE warr.plu_no = product.plu_no) "warrantyInfo", mm.PROMOTE_NO "promoteNo", MM_TYPE "mmType", PROMOTE_NAME "promoteName", b_date "beginDate", e_date "endDate", TO_CHAR(TO_DATE(b_time, 'HH24:MI'), 'HH24MI') "beginTime", TO_CHAR(TO_DATE(e_time, 'HH24:MI'), 'HH24MI') "endTime", NVL(week_1, 0)||NVL(week_2, 0)||NVL(week_3, 0)||NVL(week_4, 0)||NVL(week_5, 0)||NVL(week_6, 0)||NVL(week_7, 0) "wkDaysApplied", AMT_TYPE "amtType", MM_AMOUNT "mmAmount", GROUP_1_QTY "group1Qty", GROUP_2_QTY "group2Qty", GROUP_3_QTY "group3Qty", GROUP_4_QTY "group4Qty", GROUP_5_QTY "group5Qty", GROUP_6_QTY "group6Qty", DISC_ID "discId", G1_SHAREVALUE "g1ShareValue", G2_SHAREVALUE "g2ShareValue", G3_SHAREVALUE "g3ShareValue", G4_SHAREVALUE "g4ShareValue", G5_SHAREVALUE "g5ShareValue", G6_SHAREVALUE "g6ShareValue", STORE_BOUNDS "storeBounds", DEGREE_AMT1 "degreeAmt1", DEGREE_AMT2 "degreeAmt2", DEGREE_AMT3 "degreeAmt3", DEGREE_AMT4 "degreeAmt4", DEGREE_AMT5 "degreeAmt5", LIMITED "limited", if_equ_div "shareEquFlag", limit_type "limitType", mm_cnt_qty "limitQTY", "mmPluNo","groupNo","dmPrice", "mmGroupType", "priceType", "returnRate", "discAmt", "mmPluName", "mmPluPrice","mmPluVipPrice", "mmPluSpec", "mmPluWarrantyInfo" FROM product JOIN pluParam on product.plu_no=pluParam.plu_no JOIN mm_plu ON mm_plu.plu_no = product.plu_no OR mm_plu.plu_no=product.prod_cate_no JOIN mm ON mm.promote_no = mm_plu.promote_no JOIN (SELECT mm_plu.promote_no "mmPluPromoteNo", product.plu_no "mmPluNo", GROUP_NO "groupNo", ROUND(DM_PRICE) "dmPrice", MM_GROUPTYPE "mmGroupType", PRICE_TYPE "priceType", RETURN_RATE "returnRate", DISC_AMT "discAmt", PLU_NAME "mmPluName", product.price "mmPluPrice", product.vip_price "mmPluVipPrice", product.PLU_SPEC "mmPluSpec", (select "warrantyInfo" FROM warr WHERE warr.plu_no = product.plu_no)  "mmPluWarrantyInfo" FROM mm_plu JOIN product ON mm_plu.plu_no = product.plu_no ) mmproduct ON mmproduct."mmPluPromoteNo" = mm.promote_no WHERE withMM=1 AND TO_CHAR(SYSDATE, 'yyyymmdd') BETWEEN b_date AND e_date AND BITAND(POWER(2, MOD(TO_CHAR(SYSDATE, 'D') +5, 7)), BIN_TO_NUM(week_7,week_6,week_5,week_4,week_3,week_2,week_1)) > 0 ) SELECT "pluNo",  "pluName", "pluSpec", "price", "vipPrice", "barcode1", "barcode2", "fbAmount", "salType", "direct", "validYear", "noDupItem", "attrib",  "measureWord", "minPrice" , "paraVal", "authModel", "warrantyInfo", "promoteNo",  "mmType", "promoteName", "beginDate", "endDate", "beginTime", "endTime", "wkDaysApplied", "amtType", "mmAmount", "group1Qty", "group2Qty", "group3Qty", "group4Qty", "group5Qty", "group6Qty", "discId", "g1ShareValue", "g2ShareValue",  "g3ShareValue",  "g4ShareValue",  "g5ShareValue", "g6ShareValue",  "storeBounds", "degreeAmt1", "degreeAmt2",  "degreeAmt3",  "degreeAmt4", "degreeAmt5", "limited", "shareEquFlag", "limitType", "limitQTY", "mmPluNo","groupNo","dmPrice", "mmGroupType", "priceType", "returnRate", "discAmt", "mmPluName", "mmPluPrice","mmPluVipPrice", "mmPluSpec", "mmPluWarrantyInfo" FROM pluPromo UNION ALL SELECT product.plu_no "pluNo", product.plu_name "pluName", product.plu_spec "pluSpec", product.price "price", product.vip_price "vipPrice", product.barcode1 "barcode1", product.barcode2 "barcode2", product.fb_amount "fbAmount", product.sal_type "salType", product.direct "direct", product.valid_year "validYear", product.no_dup_item "noDupItem", product.attrib "attrib", product.unit "measureWord", product.min_price "minPrice", CASE  WHEN attrib='U' THEN 'WORDING='||product.plu_spec ELSE (SELECT keyvalue from para_store where keyvalue LIKE '#PRODNO=' || product.plu_no || '%' AND rownum=1) END "paraVal", product.auth_model "authModel",(select "warrantyInfo" FROM warr WHERE warr.plu_no = product.plu_no) "warrantyInfo", null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null FROM product JOIN pluParam on product.plu_no=pluParam.plu_no WHERE NOT EXISTS (select 1 from pluPromo)
16-Jun-2021 09:45:30.366 INFO [http-nio-8080-exec-9] tk3c.mpos.mailman.SCDataService.executeQuery executing query statement...
java.lang.ArrayIndexOutOfBoundsException: Index 0 out of bounds for length 0
        at tk3c.mpos.mailman.ServerErrorExceptionMapper.toResponse(ServerErrorExceptionMapper.java:21)
        at org.apache.cxf.jaxrs.utils.ExceptionUtils.convertFaultToResponse(ExceptionUtils.java:85)
        at org.apache.cxf.jaxrs.utils.JAXRSUtils.convertFaultToResponse(JAXRSUtils.java:1674)
        at org.apache.cxf.jaxrs.JAXRSInvoker.handleFault(JAXRSInvoker.java:370)
        at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:218)
        at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:104)
        at org.apache.openejb.server.cxf.rs.AutoJAXRSInvoker.invoke(AutoJAXRSInvoker.java:68)
        at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59)
        at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96)
        at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308)
        at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121)
        at org.apache.cxf.transport.http.AbstractHTTPDestination.invoke(AbstractHTTPDestination.java:267)
        at org.apache.openejb.server.cxf.rs.CxfRsHttpListener.doInvoke(CxfRsHttpListener.java:262)
        at org.apache.tomee.webservices.CXFJAXRSFilter.doFilter(CXFJAXRSFilter.java:99)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at org.apache.openejb.server.httpd.EEFilter.doFilter(EEFilter.java:65)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at org.apache.catalina.filters.HttpHeaderSecurityFilter.doFilter(HttpHeaderSecurityFilter.java:126)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at org.apache.catalina.filters.CorsFilter.handleNonCORS(CorsFilter.java:352)
        at org.apache.catalina.filters.CorsFilter.doFilter(CorsFilter.java:171)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:193)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:166)
        at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:202)
        at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96)
        at org.apache.tomee.catalina.OpenEJBValve.invoke(OpenEJBValve.java:45)
        at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541)
        at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:139)
        at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
        at org.apache.tomee.catalina.OpenEJBSecurityListener$RequestCapturer.invoke(OpenEJBSecurityListener.java:97)
        at org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:678)
        at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:74)
        at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:343)
        at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:367)
        at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
        at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:860)
        at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1598)
        at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
        at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
        at java.base/java.lang.Thread.run(Thread.java:834)
16-Jun-2021 09:45:31.172 SEVERE [http-nio-8080-exec-9] tk3c.mpos.mailman.ServerErrorExceptionMapper.toResponse SERVER ERROR: Java heap space java.lang.OutOfMemoryError: Java heap space

16-Jun-2021 09:45:31.181 INFO [http-nio-8080-exec-9] tk3c.mpos.mailman.SCDataService.closeConnection 3.datasource connection closed [DB] -> Conn # 11922515 CLOSE SUCCESS
16-Jun-2021 09:45:45.168 INFO [MQTT Call: paho24347478669] tk3c.mpos.packer.MQTTHelper.messageArrived received message tk3c/218/prns: {"PRN0": {"id": "0", "vendor": null, "model": "RP700", "state": 2, "status_value": 0, "status_text": "ONLINE"}, "PRN2": {"id": "2", "vendor": "Partner", "model": "RP700", "state": 2, "status_value": 2, "status_text": "ONLINE|DRAWER"}}
16-Jun-2021 09:45:45.169 INFO [MQTT Call: paho24347478669] tk3c.mpos.packer.MQTTHelper.messageArrived invoking message handler...tk3c.mpos.packer.PrinterProxy$1@1d368b5
16-Jun-2021 09:45:45.183 INFO [MQTT Call: paho24347478669] tk3c.mpos.packer.PrinterProxy$1.handle updated printer status PRN0:ONLINE PRN2:ONLINE|DRAWER
16-Jun-2021 09:45:48.197 WARNING [commons-pool-evictor-thread] org.apache.tomcat.dbcp.dbcp2.SwallowedExceptionLogger.onSwallowException An internal object pool swallowed an Exception.
        java.sql.SQLException: Protocol violation: [74]
                at oracle.jdbc.driver.T4CTTIfun.receive(T4CTTIfun.java:527)
                at oracle.jdbc.driver.T4CTTIfun.doRPC(T4CTTIfun.java:227)
                at oracle.jdbc.driver.T4C7Ocommoncall.doOLOGOFF(T4C7Ocommoncall.java:61)
                at oracle.jdbc.driver.T4CConnection.logoff(T4CConnection.java:574)
                at oracle.jdbc.driver.PhysicalConnection.close(PhysicalConnection.java:4011)
                at org.apache.tomcat.dbcp.dbcp2.DelegatingConnection.closeInternal(DelegatingConnection.java:238)
                at org.apache.tomcat.dbcp.dbcp2.PoolableConnection.reallyClose(PoolableConnection.java:232)
                at org.apache.tomcat.dbcp.dbcp2.PoolableConnectionFactory.destroyObject(PoolableConnectionFactory.java:137)
                at org.apache.tomcat.dbcp.pool2.impl.GenericObjectPool.destroy(GenericObjectPool.java:939)
                at org.apache.tomcat.dbcp.pool2.impl.GenericObjectPool.evict(GenericObjectPool.java:793)
                at org.apache.tomcat.dbcp.pool2.impl.BaseGenericObjectPool$Evictor.run(BaseGenericObjectPool.java:1138)
                at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
                at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:305)
                at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305)
                at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
                at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
                at java.base/java.lang.Thread.run(Thread.java:834)