-
Type:
Bug
-
Status: Closed (View Workflow)
-
Priority:
Critical
-
Resolution: Fixed
-
Affects Version/s: 0.20 Larks
-
Fix Version/s: 0.20 Larks
-
Labels:None
We have a risk that our server will die in prod because of intensive use of some features. We have mis-proportioned thread pool(max=200) and connection pool(max=30) configured. They should have more or less the same sizes. Here is what's happening of someone DDOSes us (more than 100 threads waiting for connection from connection pool):
Thread 4825: (state = BLOCKED) - java.lang.Object.wait(long) @bci=0 (Interpreted frame) - com.mchange.v2.resourcepool.BasicResourcePool.awaitAvailable(long) @bci=171, line=1315 (Interpreted frame) - com.mchange.v2.resourcepool.BasicResourcePool.prelimCheckoutResource(long) @bci=231, line=557 (Interpreted frame) - com.mchange.v2.resourcepool.BasicResourcePool.checkoutResource(long) @bci=2, line=477 (Interpreted frame) - com.mchange.v2.c3p0.impl.C3P0PooledConnectionPool.checkoutPooledConnection() @bci=9, line=525 (Interpreted frame) - com.mchange.v2.c3p0.impl.AbstractPoolBackedDataSource.getConnection() @bci=7, line=128 (Interpreted frame) - org.springframework.jdbc.datasource.LazyConnectionDataSourceProxy$LazyConnectionInvocationHandler.getTargetConnection(java.lang.reflect.Method) @bci=88, line=403 (Compiled frame) - org.springframework.jdbc.datasource.LazyConnectionDataSourceProxy$LazyConnectionInvocationHandler.invoke(java.lang.Object, java.lang.reflect.Method, java.lang.Object[]) @bci=423, line=376 (Compiled frame) - $Proxy13.prepareStatement(java.lang.String) @bci=16 (Compiled frame) - org.hibernate.jdbc.AbstractBatcher.getPreparedStatement(java.sql.Connection, java.lang.String, boolean, boolean, java.lang.String[], org.hibernate.ScrollMode, boolean) @bci=185, line=534 (Compiled frame) - org.hibernate.jdbc.AbstractBatcher.getPreparedStatement(java.sql.Connection, java.lang.String, boolean, org.hibernate.ScrollMode) @bci=9, line=452 (Compiled frame) - org.hibernate.jdbc.AbstractBatcher.prepareQueryStatement(java.lang.String, boolean, org.hibernate.ScrollMode) @bci=15, line=161 (Compiled frame) - org.hibernate.loader.Loader.prepareQueryStatement(org.hibernate.engine.QueryParameters, boolean, org.hibernate.engine.SessionImplementor) @bci=250, line=1700 (Compiled frame) - org.hibernate.loader.Loader.doQuery(org.hibernate.engine.SessionImplementor, org.hibernate.engine.QueryParameters, boolean) @bci=63, line=801 (Compiled frame) - org.hibernate.loader.Loader.doQueryAndInitializeNonLazyCollections(org.hibernate.engine.SessionImplementor, org.hibernate.engine.QueryParameters, boolean) @bci=60, line=274 (Compiled frame) - org.hibernate.loader.Loader.doList(org.hibernate.engine.SessionImplementor, org.hibernate.engine.QueryParameters) @bci=31, line=2533 (Interpreted frame) - org.hibernate.loader.Loader.listIgnoreQueryCache(org.hibernate.engine.SessionImplementor, org.hibernate.engine.QueryParameters) @bci=4, line=2276 (Interpreted frame) - org.hibernate.loader.Loader.list(org.hibernate.engine.SessionImplementor, org.hibernate.engine.QueryParameters, java.util.Set, org.hibernate.type.Type[]) @bci=47, line=2271 (Interpreted frame) - org.hibernate.loader.hql.QueryLoader.list(org.hibernate.engine.SessionImplementor, org.hibernate.engine.QueryParameters) @bci=19, line=452 (Interpreted frame) - org.hibernate.hql.ast.QueryTranslatorImpl.list(org.hibernate.engine.SessionImplementor, org.hibernate.engine.QueryParameters) @bci=141, line=363 (Interpreted frame) - org.hibernate.engine.query.HQLQueryPlan.performList(org.hibernate.engine.QueryParameters, org.hibernate.engine.SessionImplementor) @bci=200, line=196 (Compiled frame) - org.hibernate.impl.SessionImpl.list(java.lang.String, org.hibernate.engine.QueryParameters) @bci=49, line=1268 (Interpreted frame) - org.hibernate.impl.QueryImpl.list() @bci=27, line=102 (Interpreted frame) - org.hibernate.impl.AbstractQueryImpl.uniqueResult() @bci=1, line=890 (Interpreted frame) - org.jtalks.jcommune.model.dao.hibernate.PropertyHibernateDao.getByName(java.lang.String) @bci=19, line=45 (Interpreted frame) - sun.reflect.GeneratedMethodAccessor216.invoke(java.lang.Object, java.lang.Object[]) @bci=40 (Interpreted frame) - sun.reflect.DelegatingMethodAccessorImpl.invoke(java.lang.Object, java.lang.Object[]) @bci=6, line=25 (Compiled frame) - java.lang.reflect.Method.invoke(java.lang.Object, java.lang.Object[]) @bci=161, line=597 (Compiled frame) - org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(java.lang.Object, java.lang.reflect.Method, java.lang.Object[]) @bci=7, line=319 (Compiled frame) - org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint() @bci=12, line=183 (Compiled frame) - org.springframework.aop.framework.ReflectiveMethodInvocation.proceed() @bci=19, line=150 (Compiled frame) - org.springframework.dao.support.PersistenceExceptionTranslationInterceptor.invoke(org.aopalliance.intercept.MethodInvocation) @bci=1, line=155 (Interpreted frame) - org.springframework.aop.framework.ReflectiveMethodInvocation.proceed() @bci=101, line=172 (Compiled frame) - org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(org.aopalliance.intercept.MethodInvocation) @bci=18, line=90 (Compiled frame) - org.springframework.aop.framework.ReflectiveMethodInvocation.proceed() @bci=101, line=172 (Compiled frame) - org.springframework.aop.framework.JdkDynamicAopProxy.invoke(java.lang.Object, java.lang.reflect.Method, java.lang.Object[]) @bci=321, line=202 (Compiled frame) - $Proxy48.getByName(java.lang.String) @bci=16 (Interpreted frame) - org.jtalks.jcommune.model.entity.JCommuneProperty.getValue() @bci=15, line=78 (Interpreted frame) - org.jtalks.jcommune.model.entity.JCommuneProperty.intValue() @bci=1, line=136 (Interpreted frame) - org.jtalks.jcommune.web.listeners.SessionSetupListener.sessionCreated(javax.servlet.http.HttpSessionEvent) @bci=33, line=45 (Interpreted frame) - org.apache.catalina.session.StandardSession.tellNew() @bci=87, line=392 (Compiled frame) - org.apache.catalina.session.StandardSession.setId(java.lang.String, boolean) @bci=51, line=363 (Interpreted frame) - org.apache.catalina.session.StandardSession.setId(java.lang.String) @bci=3, line=345 (Interpreted frame) - org.apache.catalina.session.ManagerBase.createSession(java.lang.String) @bci=49, line=906 (Interpreted frame) - org.apache.catalina.session.StandardManager.createSession(java.lang.String) @bci=55, line=292 (Interpreted frame) - org.apache.catalina.connector.Request.doGetSession(boolean) @bci=246, line=2448 (Compiled frame) - org.apache.catalina.connector.Request.getSession(boolean) @bci=2, line=2157 (Compiled frame) - org.apache.catalina.connector.RequestFacade.getSession(boolean) @bci=50, line=833 (Compiled frame) - org.apache.catalina.connector.RequestFacade.getSession() @bci=25, line=844 (Interpreted frame) - javax.servlet.http.HttpServletRequestWrapper.getSession() @bci=4, line=224 (Interpreted frame) - javax.servlet.http.HttpServletRequestWrapper.getSession() @bci=4, line=224 (Interpreted frame) - org.springframework.web.servlet.mvc.method.annotation.ServletRequestMethodArgumentResolver.resolveArgument(org.springframework.core.MethodParameter, org.springframework.web.method.support.ModelAndViewContainer, org.springframework.web.context.request.NativeWebRequest, org.springframework.web.bind.support.WebDataBinderFactory) @bci=119, line=90 (Interpreted frame) - org.springframework.web.method.support.HandlerMethodArgumentResolverComposite.resolveArgument(org.springframework.core.MethodParameter, org.springframework.web.method.support.ModelAndViewContainer, org.springframework.web.context.request.NativeWebRequest, org.springframework.web.bind.support.WebDataBinderFactory) @bci=46, line=75 (Compiled frame) - org.springframework.web.method.support.InvocableHandlerMethod.getMethodArgumentValues(org.springframework.web.context.request.NativeWebRequest, org.springframework.web.method.support.ModelAndViewContainer, java.lang.Object[]) @bci=100, line=156 (Compiled frame) - org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(org.springframework.web.context.request.NativeWebRequest, org.springframework.web.method.support.ModelAndViewContainer, java.lang.Object[]) @bci=4, line=117 (Interpreted frame) - org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(org.springframework.web.context.request.NativeWebRequest, org.springframework.web.method.support.ModelAndViewContainer, java.lang.Object[]) @bci=4, line=96 (Interpreted frame) - org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse, org.springframework.web.method.HandlerMethod) @bci=85, line=617 (Interpreted frame) - org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse, org.springframework.web.method.HandlerMethod) @bci=85, line=578 (Interpreted frame) - org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse, java.lang.Object) @bci=7, line=80 (Compiled frame) - org.springframework.web.servlet.DispatcherServlet.doDispatch(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse) @bci=279, line=923 (Compiled frame) - org.springframework.web.servlet.DispatcherServlet.doService(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse) @bci=292, line=852 (Interpreted frame) - org.springframework.web.servlet.FrameworkServlet.processRequest(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse) @bci=111, line=882 (Interpreted frame) - org.springframework.web.servlet.FrameworkServlet.doGet(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse) @bci=3, line=778 (Interpreted frame) - javax.servlet.http.HttpServlet.service(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse) @bci=35, line=617 (Interpreted frame) - javax.servlet.http.HttpServlet.service(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=30, line=717 (Interpreted frame) - org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=376, line=290 (Interpreted frame) - org.apache.catalina.core.ApplicationFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=101, line=206 (Interpreted frame) - org.springframework.web.filter.HiddenHttpMethodFilter.doFilterInternal(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse, javax.servlet.FilterChain) @bci=71, line=77 (Interpreted frame) - org.springframework.web.filter.OncePerRequestFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=90, line=76 (Interpreted frame) - org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=119, line=235 (Interpreted frame) - org.apache.catalina.core.ApplicationFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=101, line=206 (Interpreted frame) - com.opensymphony.sitemesh.webapp.SiteMeshFilter.obtainContent(com.opensymphony.sitemesh.ContentProcessor, com.opensymphony.sitemesh.webapp.SiteMeshWebAppContext, javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse, javax.servlet.FilterChain) @bci=18, line=129 (Interpreted frame) - com.opensymphony.sitemesh.webapp.SiteMeshFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=126, line=77 (Interpreted frame) - org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=119, line=235 (Interpreted frame) - org.apache.catalina.core.ApplicationFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=101, line=206 (Compiled frame) - org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=68, line=368 (Compiled frame) - org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(org.springframework.security.web.FilterInvocation) @bci=87, line=109 (Interpreted frame) - org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=15, line=83 (Interpreted frame) - org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=194, line=380 (Compiled frame) - org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=17, line=97 (Interpreted frame) - org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=194, line=380 (Compiled frame) - org.springframework.security.web.session.SessionManagementFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=294, line=100 (Interpreted frame) - org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=194, line=380 (Compiled frame) - org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=148, line=78 (Interpreted frame) - org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=194, line=380 (Compiled frame) - org.springframework.security.web.authentication.rememberme.RememberMeAuthenticationFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=244, line=112 (Interpreted frame) - org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=194, line=380 (Compiled frame) - org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=17, line=54 (Interpreted frame) - org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=194, line=380 (Compiled frame) - org.springframework.security.web.savedrequest.RequestCacheAwareFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=32, line=35 (Interpreted frame) - org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=194, line=380 (Compiled frame) - org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=414, line=177 (Interpreted frame) - org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=194, line=380 (Compiled frame) - org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=28, line=187 (Interpreted frame) - org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=194, line=380 (Compiled frame) - org.springframework.security.web.authentication.logout.LogoutFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=149, line=105 (Interpreted frame) - org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=194, line=380 (Compiled frame) - org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=165, line=79 (Interpreted frame) - org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=194, line=380 (Compiled frame) - org.springframework.security.web.FilterChainProxy.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=146, line=169 (Interpreted frame) - org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(javax.servlet.Filter, javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=5, line=346 (Interpreted frame) - org.springframework.web.filter.DelegatingFilterProxy.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=71, line=259 (Interpreted frame) - org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=119, line=235 (Interpreted frame) - org.apache.catalina.core.ApplicationFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=101, line=206 (Interpreted frame) - org.springframework.orm.hibernate3.support.OpenSessionInViewFilter.doFilterInternal(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse, javax.servlet.FilterChain) @bci=89, line=198 (Interpreted frame) - org.springframework.web.filter.OncePerRequestFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=90, line=76 (Interpreted frame) - org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=119, line=235 (Interpreted frame) - org.apache.catalina.core.ApplicationFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=101, line=206 (Interpreted frame) - org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(javax.servlet.http.HttpServletRequest, javax.servlet.http.HttpServletResponse, javax.servlet.FilterChain) @bci=53, line=88 (Interpreted frame) - org.springframework.web.filter.OncePerRequestFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=90, line=76 (Interpreted frame) - org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=119, line=235 (Interpreted frame) - org.apache.catalina.core.ApplicationFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=101, line=206 (Interpreted frame) - com.cj.trim.trimFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=51 (Compiled frame) - org.jtalks.jcommune.web.filters.TrimFilter.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse, javax.servlet.FilterChain) @bci=27, line=82 (Interpreted frame) - org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=119, line=235 (Interpreted frame) - org.apache.catalina.core.ApplicationFilterChain.doFilter(javax.servlet.ServletRequest, javax.servlet.ServletResponse) @bci=101, line=206 (Interpreted frame) - org.apache.catalina.core.StandardWrapperValve.invoke(org.apache.catalina.connector.Request, org.apache.catalina.connector.Response) @bci=804, line=233 (Interpreted frame) - org.apache.catalina.core.StandardContextValve.invoke(org.apache.catalina.connector.Request, org.apache.catalina.connector.Response) @bci=365, line=191 (Interpreted frame) - org.apache.catalina.core.StandardHostValve.invoke(org.apache.catalina.connector.Request, org.apache.catalina.connector.Response) @bci=64, line=127 (Interpreted frame) - org.apache.catalina.valves.ErrorReportValve.invoke(org.apache.catalina.connector.Request, org.apache.catalina.connector.Response) @bci=6, line=103 (Interpreted frame) - org.apache.catalina.core.StandardEngineValve.invoke(org.apache.catalina.connector.Request, org.apache.catalina.connector.Response) @bci=42, line=109 (Interpreted frame) - org.apache.catalina.connector.CoyoteAdapter.service(org.apache.coyote.Request, org.apache.coyote.Response) @bci=158, line=293 (Interpreted frame) - org.apache.coyote.http11.Http11Processor.process(java.net.Socket) @bci=531, line=861 (Interpreted frame) - org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(java.net.Socket) @bci=82, line=606 (Interpreted frame) - org.apache.tomcat.util.net.JIoEndpoint$Worker.run() @bci=41, line=489 (Interpreted frame) - java.lang.Thread.run() @bci=11, line=662 (Interpreted frame)