JVM crash

[sudipto shekhar]

Hi Folks,

I am using "Java VM: Java HotSpot(TM) Server VM (1.4.2_05-b04 mixed mode) "

The JVM crashed and produced the following output in the hs_err_pid error file:

An unexpected exception has been detected in native code outside the VM.
Unexpected Signal : 10 occurred at PC=0xFF3A062C
Function=memcpy+0x28
Library=/usr/platform/SUNW,Sun-Fire-V240/lib/libc_psr.so.1

Current Java thread:
at java.util.zip.ZipFile.read(Native Method)
at java.util.zip.ZipFile.access$1000(ZipFile.java:26)
at java.util.zip.ZipFile$ZipFileInputStream.read(ZipFile.java:433)
- locked (a java.util.jar.JarFile)
at java.util.zip.ZipFile$1.fill(ZipFile.java:228)
at java.util.zip.InflaterInputStream.read(InflaterInputStream.java:134)
at java.io.FilterInputStream.read(FilterInputStream.java:111)
at java.io.PushbackInputStream.read(PushbackInputStream.java:161)
at weblogic.xml.babel.reader.XmlReader$Utf8Reader.read(XmlReader.java:656)
at weblogic.xml.babel.reader.XmlReader.read(XmlReader.java:492)
at weblogic.xml.babel.scanner.ScannerState.read(ScannerState.java:403)
at weblogic.xml.babel.scanner.ScannerState.skipSpace(ScannerState.java:453)
at weblogic.xml.babel.scanner.OpenTag.read(OpenTag.java:32)
at weblogic.xml.babel.scanner.Scanner.startState(Scanner.java:251)
at weblogic.xml.babel.scanner.Scanner.scan(Scanner.java:178)
at weblogic.xml.babel.baseparser.BaseParser.accept(BaseParser.java:511)
at weblogic.xml.babel.baseparser.StartElement.parse(StartElement.java:77)
at weblogic.xml.babel.baseparser.BaseParser.parseElement(BaseParser.java:441)
at weblogic.xml.babel.baseparser.BaseParser.parseSome(BaseParser.java:318)
at weblogic.xml.babel.parsers.XMLInputStreamParser.internalParseSome(XMLInputStreamParser.java:124)
at weblogic.xml.babel.parsers.XMLInputStreamParser.streamParseSome(XMLInputStreamParser.java:139)
at weblogic.xml.babel.stream.XMLInputStreamBase.peek(XMLInputStreamBase.java:221)
at weblogic.xml.stream.util.XMLInputStreamFilterBase.hasNext(XMLInputStreamFilterBase.java:81)
at weblogic.xml.xmlnode.XMLNode.readChildren(XMLNode.java:1044)
at weblogic.xml.xmlnode.XMLNode.read(XMLNode.java:741)
at weblogic.xml.xmlnode.XMLNode.readChildren(XMLNode.java:1053)
at weblogic.xml.xmlnode.XMLNode.read(XMLNode.java:741)
at weblogic.xml.xmlnode.XMLNode.readChildren(XMLNode.java:1053)
at weblogic.xml.xmlnode.XMLNode.read(XMLNode.java:741)
at weblogic.webservice.dd.ParsingHelper.forkSubtrees(ParsingHelper.java:273)
at weblogic.webservice.dd.DDLoader.processTypesElement(DDLoader.java:535)
at weblogic.webservice.dd.DDLoader.processWebServiceElement(DDLoader.java:488)
at weblogic.webservice.dd.DDLoader.processWebServiceElements(DDLoader.java:310)
at weblogic.webservice.dd.DDLoader.processWebServicesElement(DDLoader.java:298)
at weblogic.webservice.dd.DDLoader.load(DDLoader.java:276)
at weblogic.webservice.dd.DDLoader.load(DDLoader.java:253)
at weblogic.servlet.internal.dd.DescriptorLoader.initializeWebServices(DescriptorLoader.java:772)
at weblogic.servlet.internal.dd.DescriptorLoader.(DescriptorLoader.java:458)
at weblogic.servlet.internal.dd.DescriptorLoader.(DescriptorLoader.java:333)
at weblogic.management.mbeans.custom.WebAppComponent.readDescriptor(WebAppComponent.java:264)
at weblogic.management.mbeans.custom.Component.initializeTwoPhase(Component.java:411)
at weblogic.management.mbeans.custom.Component.initialize(Component.java:283)
at weblogic.management.mbeans.custom.WebAppComponent.findOrCreateWebDescriptor(WebAppComponent.java:109)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:324)
at weblogic.management.internal.DynamicMBeanImpl.invokeLocally(DynamicMBeanImpl.java:755)
at weblogic.management.internal.DynamicMBeanImpl.invoke(DynamicMBeanImpl.java:734)
at weblogic.management.internal.ConfigurationMBeanImpl.invoke(ConfigurationMBeanImpl.java:519)
at com.sun.management.jmx.MBeanServerImpl.invoke(MBeanServerImpl.java:1557)
at com.sun.management.jmx.MBeanServerImpl.invoke(MBeanServerImpl.java:1525)
at weblogic.management.internal.RemoteMBeanServerImpl.private_invoke(RemoteMBeanServerImpl.java:990)
at weblogic.management.internal.RemoteMBeanServerImpl.invoke(RemoteMBeanServerImpl.java:948)
at weblogic.management.internal.MBeanProxy.invoke(MBeanProxy.java:948)
at weblogic.management.internal.MBeanProxy.invokeForCachingStub(MBeanProxy.java:475)
at weblogic.management.configuration.WebServiceComponentMBean_Stub.findOrCreateWebDescriptor(WebServiceComponentMBean_Stub.java:3249)
at weblogic.management.console.webapp._domain.__webservicecomponent.emitTestUrls(__webservicecomponent.java:154)
at weblogic.management.console.webapp._domain.__webservicecomponent._jspService(__webservicecomponent.java:2939)
at weblogic.servlet.jsp.JspBase.service(JspBase.java:33)
at weblogic.servlet.internal.ServletStubImpl$ServletInvocationAction.run(ServletStubImpl.java:1077)
at weblogic.servlet.internal.ServletStubImpl.invokeServlet(ServletStubImpl.java:465)
at weblogic.servlet.internal.ServletStubImpl.invokeServlet(ServletStubImpl.java:348)
at weblogic.servlet.internal.RequestDispatcherImpl.forward(RequestDispatcherImpl.java:330)
at weblogic.servlet.jsp.PageContextImpl.forward(PageContextImpl.java:150)
at weblogic.management.console.actions.ForwardAction.perform(ForwardAction.java:35)
at weblogic.management.console.actions.internal.ActionServlet.doAction(ActionServlet.java:182)
at weblogic.management.console.actions.internal.ActionServlet.doGet(ActionServlet.java:92)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:740)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:853)
at weblogic.servlet.internal.ServletStubImpl$ServletInvocationAction.run(ServletStubImpl.java:1077)
at weblogic.servlet.internal.ServletStubImpl.invokeServlet(ServletStubImpl.java:465)
at weblogic.servlet.internal.ServletStubImpl.invokeServlet(ServletStubImpl.java:348)
at weblogic.servlet.internal.WebAppServletContext$ServletInvocationAction.run(WebAppServletContext.java:7047)
at weblogic.security.acl.internal.AuthenticatedSubject.doAs(AuthenticatedSubject.java:321)
at weblogic.security.service.SecurityManager.runAs(SecurityManager.java:121)
at weblogic.servlet.internal.WebAppServletContext.invokeServlet(WebAppServletContext.java:3902)
at weblogic.servlet.internal.ServletRequestImpl.execute(ServletRequestImpl.java:2773)
at weblogic.kernel.ExecuteThread.execute(ExecuteThread.java:224)
at weblogic.kernel.ExecuteThread.run(ExecuteThread.java:183)

Dynamic libraries:
0x10000 /apps/prod/bea81/jdk14/bin/java
0xff360000 /usr/lib/libthread.so.1
0xff350000 /usr/lib/libdl.so.1
0xff280000 /usr/lib/libc.so.1
0xff3a0000 /usr/platform/SUNW,Sun-Fire-V240/lib/libc_psr.so.1
0xfe000000 /apps/prod/bea81/jdk14/jre/lib/sparc/server/libjvm.so
0xff210000 /usr/lib/libCrun.so.1
0xff1f0000 /usr/lib/libsocket.so.1
0xff100000 /usr/lib/libnsl.so.1
0xff0b0000 /usr/lib/libm.so.1
0xff1c0000 /usr/lib/libsched.so.1
0xff090000 /usr/lib/libmp.so.2
0xff060000 /apps/prod/bea81/jdk14/jre/lib/sparc/native_threads/libhpi.so
0xff030000 /apps/prod/bea81/jdk14/jre/lib/sparc/libverify.so
0xfe7c0000 /apps/prod/bea81/jdk14/jre/lib/sparc/libjava.so
0xfe7a0000 /apps/prod/bea81/jdk14/jre/lib/sparc/libzip.so
0xfe5f0000 /usr/lib/locale/en_US.ISO8859-1/en_US.ISO8859-1.so.2
0xfba90000 /apps/prod/bea81/jdk14/jre/lib/sparc/libnet.so
0xfb890000 /apps/prod/bea81/weblogic81/server/lib/solaris/libweblogicunix1.so
0xfb870000 /apps/prod/bea81/jdk14/jre/lib/sparc/libnio.so
0xfb850000 /usr/lib/librt.so.1
0xfb830000 /usr/lib/libaio.so.1
0xfb810000 /usr/lib/libsendfile.so.1
0xf9560000 /apps/prod/bea81/jdk14/jre/lib/sparc/libioser12.so
0xf9460000 /apps/prod/bea81/weblogic81/server/lib/solaris/libstackdump.so
0xf9440000 /apps/prod/bea81/weblogic81/server/lib/solaris/libmuxer.so
0xf9420000 /usr/ucblib/libucb.so.1
0xe4bb0000 /usr/lib/libresolv.so.2
0xe4950000 /usr/lib/libelf.so.1

Heap at VM Abort:
Heap
def new generation total 22528K, used 8809K [0xe4c00000, 0xe62a0000, 0xe8ea0000)
eden space 21888K, 37% used [0xe4c00000, 0xe53fa7d8, 0xe6160000)
from space 640K, 100% used [0xe6160000, 0xe6200000, 0xe6200000)
to space 640K, 0% used [0xe6200000, 0xe6200000, 0xe62a0000)
tenured generation total 46240K, used 31737K [0xe8ea0000, 0xebbc8000, 0xf1400000)
the space 46240K, 68% used [0xe8ea0000, 0xead9e438, 0xead9e600, 0xebbc8000)
compacting perm gen total 32256K, used 31974K [0xf1400000, 0xf3380000, 0xf9400000)
the space 32256K, 99% used [0xf1400000, 0xf3339b78, 0xf3339c00, 0xf3380000)

Local Time = Thu May 6 00:56:28 2010
Elapsed Time = 663359
#
# The exception above was detected in native code outside the VM
#
# Java VM: Java HotSpot(TM) Server VM (1.4.2_05-b04 mixed mode)
#

Could somebody help me with this?

What is the exact reason for crash ?

Also I see that the permanent generation used is 99% .

Does this relate to the crash ?

Does every crash results after 99% permanent generation usage?

Is there a patch to fix this?

The OS is Solaris 8.

Moderator please shift this to the correct forum if this is not the right one for this post!!

Thanks.

[Jesper de Jong]

Looking at the stack trace, it looks like this happens in a native library while reading a ZIP file.

I'm not a Solaris expert, but maybe you need a bugfix patch for Solaris to solve this. Try contacting the vendor of your server (Oracle, formerly Sun) and ask for support.

[sudipto shekhar]

Hey Jasper,

Thank you for the answer.

I have seen a similar crash and the similarity that I am talking about is this:

compacting perm gen total 32256K, used 31974K [0xf1400000, 0xf3380000, 0xf9400000)
the space 32256K, 99% used [0xf1400000, 0xf3339b78, 0xf3339c00, 0xf3380000)

As we can see the perm gen space usage is 99%!!

This is similar to the other crash that I am talking about.

Is there a relation between JVM crash and the perm gen usage to 99%?

I was wondering if you could throw any light on this.

[Rob Spoor]

Most crashes caused by native code are memory related. This could very well be the case here. Perhaps the native code tries to allocate a piece of memory, this fails due to no more memory being available, and the native code does not check if the allocation succeeded or not.

[sudipto shekhar]

Hey Rob,

Good to see you and Thanks for the reply.

Yes I completely agree with you that there is a native call. The native call failed as the call might be trying to allocate some memory which JVM could not find and hence the JVM crashed!

But is there a relation between the crash and the Permanent Generation being 99%.

As you know Permanent Generation is the memory allocation outside of the JVM and is only used to write the class loading part.
We also know that during runtime there is almost no class loading!

So why is the Permanent Generation getting filled up to 99% as the JVM crashes.

I said that I have see another crash file where during the crash the perm. gen was filled up to 99%. The final part of the crash file is as follows:

Heap
def new generation total 104832K, used 82053K [0x87c00000, 0x8edc0000, 0x8edc0000)
eden space 93184K, 84% used [0x87c00000, 0x8c8cd098, 0x8d700000)
from space 11648K, 29% used [0x8e260000, 0x8e5b4600, 0x8edc0000)
to space 11648K, 0% used [0x8d700000, 0x8d700000, 0x8e260000)
tenured generation total 932096K, used 87413K [0x8edc0000, 0xc7c00000, 0xc7c00000)
the space 932096K, 9% used [0x8edc0000, 0x9431d770, 0x9431d800, 0xc7c00000)
compacting perm gen total 136704K, used 136685K [0xc7c00000, 0xd0180000, 0xd7c00000)
the space 136704K, 99% used [0xc7c00000, 0xd017b700, 0xd017b800, 0xd0180000)
ro space 8192K, 80% used [0xd7c00000, 0xd82732e8, 0xd8273400, 0xd8400000)
rw space 12288K, 59% used [0xd8400000, 0xd8b302d8, 0xd8b30400, 0xd9000000)

Dynamic libraries:
0x00010000 /herramientas/servers/bea/wls103/jdk160_05/bin/java
0xff3a0000 /lib/libthread.so.1
0xff370000 /herramientas/servers/bea/wls103/jdk160_05/bin/../jre/lib/sparc/jli/libjli.so
0xff350000 /lib/libdl.so.1
0xff200000 /lib/libc.so.1
0xff340000 /platform/SUNW,SPARC-Enterprise/lib/libc_psr.so.1
0xfe800000 /herramientas/servers/bea/wls103/jdk160_05/jre/lib/sparc/client/libjvm.so
0xff310000 /lib/libsocket.so.1
0xff330000 /usr/lib/libsched.so.1
0xff1e0000 /lib/libm.so.1
0xff1b0000 /usr/lib/libCrun.so.1
0xff190000 /lib/libdoor.so.1
0xff080000 /lib/libnsl.so.1
0xfef80000 /lib/libm.so.2
0xff160000 /lib/libscf.so.1
0xff140000 /lib/libuutil.so.1
0xff060000 /lib/libgen.so.1
0xfef50000 /lib/libmd.so.1
0xfef30000 /lib/libmp.so.2
0xfeed0000 /herramientas/servers/bea/wls103/jdk160_05/jre/lib/sparc/native_threads/libhpi.so
0xfee70000 /herramientas/servers/bea/wls103/jdk160_05/jre/lib/sparc/libverify.so
0xfe7c0000 /herramientas/servers/bea/wls103/jdk160_05/jre/lib/sparc/libjava.so
0xfe7a0000 /herramientas/servers/bea/wls103/jdk160_05/jre/lib/sparc/libzip.so
0xfe6e0000 /usr/lib/locale/en_US.ISO8859-15/en_US.ISO8859-15.so.3
0xfaed0000 /herramientas/servers/bea/wls103/jdk160_05/jre/lib/sparc/libmanagement.so
0xfaea0000 /herramientas/servers/bea/wls103/jdk160_05/jre/lib/sparc/libnet.so
0xfadb0000 /herramientas/servers/bea/wls103/jdk160_05/jre/lib/sparc/libnio.so
0xfad90000 /lib/librt.so.1
0xfad70000 /lib/libaio.so.1
0xfad50000 /usr/lib/libsendfile.so.1
0xfad20000 /herramientas/servers/bea/wls103/jdk160_05/jre/lib/sparc/libj2pkcs11.so
0xfacf0000 /usr/lib/libpkcs11.so
0xfacd0000 /usr/lib/libcryptoutil.so.1
0xfac70000 /usr/lib/security/pkcs11_softtoken.so
0xfac50000 /herramientas/servers/bea/wls103/wlserver_10.3/server/native/solaris/sparc/libstackdump.so
0xfac30000 /herramientas/servers/bea/wls103/wlserver_10.3/server/native/solaris/sparc/libwlfileio2.so
0xfac10000 /herramientas/servers/bea/wls103/wlserver_10.3/server/native/solaris/sparc/libmuxer.so
0xfab60000 /usr/ucblib/libucb.so.1
0xfaa20000 /lib/libresolv.so.2
0xfab30000 /lib/libelf.so.1
0xf8c10000 /herramientas/servers/bea/wls103/jdk160_05/jre/lib/sparc/librmi.so

VM Arguments:
jvm_args: -Xms1024m -Xmx1024m -XX:MaxPermSize=256m -Xverify:none -da -Dplatform.home=/herramientas/servers/bea/wls103/wlserver_10.3 -Dwls.home=/herramientas/servers/bea/wls103/wlserver_10.3/server -Dweblogic.home=/herramientas/servers/bea/wls103/wlserver_10.3/server -Dweblogic.management.discover=true -Dwlw.iterativeDev= -Dwlw.testConsole= -Dwlw.logErrorsToConsole= -Denlacefi.licencia.dir=/AP/herramientas/apps/intranet/indeval -Defindeval=/AP/herramientas/apps/intranet/indeval/indeval-config/efindeval.properties -Dwicket.configuration=deployment -Dweblogic.ext.dirs=/herramientas/servers/bea/wls103/patch_wls1030/profiles/default/sysext_manifest_classpath:/herramientas/servers/bea/wls103/patch_cie660/profiles/default/sysext_manifest_classpath -Dweblogic.Name=AdminServer -Djava.security.policy=/herramientas/servers/bea/wls103/wlserver_10.3/server/lib/weblogic.policy
java_command: weblogic.Server
Launcher Type: SUN_STANDARD

Environment Variables:
JAVA_HOME=/herramientas/servers/bea/wls103/jdk160_05
CLASSPATH=:/herramientas/servers/bea/wls103/patch_wls1030/profiles/default/sys_manifest_classpath/weblogic_patch.jar:/herramientas/servers/bea/wls103/patch_cie660/profiles/default/sys_manifest_classpath/weblogic_patch.jar:/herramientas/servers/bea/wls103/jdk160_05/lib/tools.jar:/herramientas/servers/bea/wls103/wlserver_10.3/server/lib/weblogic_sp.jar:/herramientas/servers/bea/wls103/wlserver_10.3/server/lib/weblogic.jar:/herramientas/servers/bea/wls103/modules/features/weblogic.server.modules_10.3.0.0.jar:/herramientas/servers/bea/wls103/wlserver_10.3/server/lib/webservices.jar:/herramientas/servers/bea/wls103/modules/org.apache.ant_1.6.5/lib/ant-all.jar:/herramientas/servers/bea/wls103/modules/net.sf.antcontrib_1.0.0.0_1-0b2/lib/ant-contrib.jar::/herramientas/servers/bea/wls103/wlserver_10.3/common/eval/pointbase/lib/pbclient57.jar:/herramientas/servers/bea/wls103/wlserver_10.3/server/lib/xqrl.jar::/AP/herramientas/apps/intranet/indeval/indeval-config:/AP/herramientas/apps/intranet/indeval/indeval-config/config:
PATH=/herramientas/servers/bea/wls103/wlserver_10.3/server/bin:/herramientas/servers/bea/wls103/modules/org.apache.ant_1.6.5/bin:/herramientas/servers/bea/wls103/jdk160_05/jre/bin:/herramientas/servers/bea/wls103/jdk160_05/bin:/usr/bin:/usr/ucb:/etc:.
LD_LIBRARY_PATH=/herramientas/servers/bea/wls103/jdk160_05/jre/lib/sparc/client:/herramientas/servers/bea/wls103/jdk160_05/jre/lib/sparc:/herramientas/servers/bea/wls103/jdk160_05/jre/../lib/sparc::/herramientas/servers/bea/wls103/wlserver_10.3/server/native/solaris/sparc:/herramientas/servers/bea/wls103/wlserver_10.3/server/native/solaris/sparc/oci920_8
SHELL=/bin/ksh

Signal Handlers:
SIGSEGV: [libjvm.so+0x4faa70], sa_mask[0]=0xffbffeff, sa_flags=0x0000000c
SIGBUS: [libjvm.so+0x4faa70], sa_mask[0]=0xffbffeff, sa_flags=0x0000000c
SIGFPE: [libjvm.so+0x194504], sa_mask[0]=0xffbffeff, sa_flags=0x0000000c
SIGPIPE: [libjvm.so+0x194504], sa_mask[0]=0xffbffeff, sa_flags=0x0000000c
SIGXFSZ: [libjvm.so+0x194504], sa_mask[0]=0xffbffeff, sa_flags=0x0000000c
SIGILL: [libjvm.so+0x194504], sa_mask[0]=0xffbffeff, sa_flags=0x0000000c
SIGUSR1: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
SIGUSR2: SIG_DFL, sa_mask[0]=0x00000000, sa_flags=0x00000000
SIGQUIT: [libjvm.so+0x427378], sa_mask[0]=0xffbffeff, sa_flags=0x00000004
SIGHUP: SIG_IGN, sa_mask[0]=0x00000000, sa_flags=0x00000000
SIGINT: SIG_IGN, sa_mask[0]=0x00000000, sa_flags=0x00000000
SIGTERM: [libjvm.so+0x427378], sa_mask[0]=0xffbffeff, sa_flags=0x00000004
SIG39: [libjvm.so+0x42a4f8], sa_mask[0]=0x00000000, sa_flags=0x00000008
SIG40: [libjvm.so+0x194504], sa_mask[0]=0xffbffeff, sa_flags=0x0000000c


--------------- S Y S T E M ---------------

OS: Solaris 10 8/07 s10s_u4wos_12b SPARC
Copyright 2007 Sun Microsystems, Inc. All Rights Reserved.
Use is subject to license terms.
Assembled 16 August 2007

uname:SunOS 5.10 Generic_138888-08 sun4u (T2 libthread)
rlimit: STACK 8192k, CORE infinity, NOFILE 65536, AS infinity
load average:6.17 6.55 6.21

CPU:total 16 has_v8, has_v9, has_vis1, has_vis2, is_ultra3

Memory: 8k page, physical 67108864k(4891096k free)

vm_info: Java HotSpot(TM) Client VM (10.0-b19) for solaris-sparc JRE (1.6.0_05-b13), built on Feb 22 2008 02:00:35 by "" with Workshop 5.8

time: Mon May 10 17:23:29 2010
elapsed time: 363640 seconds