[libil2cpp.so] il2cpp::os::Image::GetELFImageBuildID()

Unity Engine
, ,
1

We’re chasing down a segfault that might be caused by 2022.3.0f1

It kind of looks similar to this (we are also using ironsource): Lot of crashes at il2cpp::vm::Object::GetClass(Il2CppObject*) 2021.3.19f1 LTS

backtrace:
#00 pc 0x0000000001111124 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libil2cpp.so (il2cpp::os::Image::GetELFImageBuildID())
#01 pc 0x000000000116cc58 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libil2cpp.so (il2cpp_native_stack_trace+132)
#02 pc 0x0000000000506799 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (scripting_stack_trace_info_for(ScriptingExceptionPtr, StackTraceInfo&))
#03 pc 0x00000000005120d5 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (Scripting::LogExceptionFromManaged(ScriptingExceptionPtr, int, char const*, bool, Scripting::LogExceptionFromMangedSettings const*))
#04 pc 0x000000000025263f /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (DebugLogHandler::Internal_LogException(ScriptingObjectPtr, Object*))
#05 pc 0x000000000025260d /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (DebugLogHandler_CUSTOM_Internal_LogException(ScriptingBackendNativeObjectPtrOpaque*, ScriptingBackendNativeObjectPtrOpaque*))
#06 pc 0x0000000003eb43d0 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libil2cpp.so (UnityLogHandlerIntegration_LogException_m23CB511805A3C839541A9868BC1FD148CF9A9E47)
#07 pc 0x00000000046a0ba0 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libil2cpp.so (Debug_CallOverridenDebugHandler_mB78E42B7792823F50830A325BB1CF50C45FEAA8B)
#08 pc 0x0000000000fd2790 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libil2cpp.so (RuntimeInvoker_FalseByte_t94D9231AC217BE4D2E004C4CD32DF6D099EA41A3_RuntimeObject_RuntimeObject(void ()(), MethodInfo const, void*, void**, void*))
#09 pc 0x000000000118121c /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libil2cpp.so (il2cpp::vm::Runtime::InvokeWithThrow(MethodInfo const*, void*, void**))
#10 pc 0x00000000011810c0 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libil2cpp.so (il2cpp::vm::Runtime::Invoke(MethodInfo const*, void*, void**, Il2CppException**))
#11 pc 0x0000000000506161 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (scripting_method_invoke(ScriptingMethodPtr, ScriptingObjectPtr, ScriptingArguments&, ScriptingExceptionPtr*, bool))
#12 pc 0x00000000005109bd /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (ScriptingInvocation::Invoke(ScriptingExceptionPtr*, bool))
#13 pc 0x0000000000512d3f /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (bool ScriptingInvocation::Invoke(ScriptingExceptionPtr*, bool))
#14 pc 0x000000000058bbdb /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (Scripting::UnityEngine::smile:ebugProxy::CallOverridenDebugHandler(ScriptingObjectPtr, Scripting::UnityEngine::ObjectProxy, ScriptingExceptionPtr*))
#15 pc 0x000000000051124b /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (Scripting::LogException(ScriptingExceptionPtr, int, char const*, bool))
#16 pc 0x00000000005109e3 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (ScriptingInvocation::Invoke(ScriptingExceptionPtr*, bool))
#17 pc 0x000000000051a38d /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (MonoBehaviour::CallUpdateMethod(int))
#18 pc 0x000000000039cdeb /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (void BaseBehaviourManager::CommonUpdate())
#19 pc 0x00000000004381e7 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (ExecutePlayerLoop(NativePlayerLoopSystem*))
#20 pc 0x0000000000438209 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (ExecutePlayerLoop(NativePlayerLoopSystem*))
#21 pc 0x00000000004383eb /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (PlayerLoop())
#22 pc 0x0000000000565333 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (UnityPlayerLoop())
#23 pc 0x0000000000578a17 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/lib/arm/libunity.so (nativeRender(_JNIEnv*, _jobject*))
#24 pc 0x00000000003c1b9d /apex/com.android.art/lib/libart.so (art_quick_generic_jni_trampoline+44)
#25 pc 0x00000000000f0dec /apex/com.android.art/lib/libart.so (nterp_helper+2908)
#26 pc 0x0000000000e565c4 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/oat/arm/base.vdex (com.unity3d.player.UnityPlayer.-$$Nest$mnativeRender)
#27 pc 0x00000000000f02c8 /apex/com.android.art/lib/libart.so (nterp_helper+56)
#28 pc 0x0000000001615f28 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/oat/arm/base.vdex (com.unity3d.player.UnityPlayer$C$a.handleMessage+244)
#29 pc 0x0000000000ae102b /data/misc/apexdata/com.android.art/dalvik-cache/arm/boot.oat (android.os.Handler.dispatchMessage+122)
#30 pc 0x0000000000ae46a7 /data/misc/apexdata/com.android.art/dalvik-cache/arm/boot.oat (android.os.Looper.loopOnce+886)
#31 pc 0x0000000000ae424b /data/misc/apexdata/com.android.art/dalvik-cache/arm/boot.oat (android.os.Looper.loop+1034)
#32 pc 0x00000000000f033c /apex/com.android.art/lib/libart.so (nterp_helper+172)
#33 pc 0x00000000006bc6b8 /data/app/~~7P2d5EO1SbsxSxi4F9JLng==/au.com.oddgames.monstertruckdestruction-MFHOw1mX5yJ8OkHOrISBfA==/oat/arm/base.vdex (com.unity3d.player.UnityPlayer$C.run+48)
#34 pc 0x00000000003bd1d5 /apex/com.android.art/lib/libart.so (art_quick_invoke_stub_internal+68)
#35 pc 0x00000000003bcc77 /apex/com.android.art/lib/libart.so (void art::quick_invoke_reg_setup(art::ArtMethod*, unsigned int*, unsigned int, art::Thread*, art::JValue*, char const*) (.__uniq.192663596067446536341070919852553954320.llvm.16200230356545185596)+158)
#36 pc 0x000000000028a4c5 /apex/com.android.art/lib/libart.so (art::ArtMethod::Invoke(art::Thread*, unsigned int*, unsigned int, art::JValue*, char const*)+136)
#37 pc 0x0000000000313969 /apex/com.android.art/lib/libart.so (art::(anonymous namespace)::InvokeWithArgArray(art::ScopedObjectAccessAlreadyRunnable const&, art::ArtMethod*, art::(anonymous namespace)::ArgArray*, art::JValue*, char const*) (.__uniq.245181933781456475607640333933569312899.llvm.2586733929373010981)+40)
#38 pc 0x0000000000313875 /apex/com.android.art/lib/libart.so (art::JValue art::InvokeVirtualOrInterfaceWithJValuesart::ArtMethod*(art::ScopedObjectAccessAlreadyRunnable const&, _jobject*, art::ArtMethod*, jvalue const*)+164)
#39 pc 0x000000000033a3af /apex/com.android.art/lib/libart.so (art::JValue art::InvokeVirtualOrInterfaceWithJValues<_jmethodID*>(art::ScopedObjectAccessAlreadyRunnable const&, _jobject*, _jmethodID*, jvalue const*)+42)
#40 pc 0x00000000002f0625 /apex/com.android.art/lib/libart.so (art::thread::CreateCallback(void*)+416)
#41 pc 0x00000000000804c7 /apex/com.android.runtime/lib/bionic/libc.so (__pthread_start(void*)+40)
#42 pc 0x0000000000039745 /apex/com.android.runtime/lib/bionic/libc.so (__start_thread+30)

2

Is the crash consistent or intermittent? The GetELFImageBuildID method is attempting to introspect the loaded images in the executable and find the build ID for each one, so that build ID can be synced up with debug symbols.

I suspect there is some case we missed during image processing which causes this crash.

3

I can’t cause it on my test devices, it’s only coming from google play at around 1% of sessions. So it’s fairly common.

Through sentry i receive the original exception, then afterwards I get that crash log as a seperate error.

4

Is there any pattern to the device or OS versions where this happens?

5

No pattern really, I’ve completely removed sentry to make sure that isn’t messing it and it isn’t.

The biggest changes between builds was updating ironsource mediation adpaters and moving from Unity 2021 to 2022.3.0 I’ve since updated iron source adapters again and removed some of them (no change) and Im about to try 2022.3.4

9113116--1264045--upload_2023-6-29_19-29-25.png
9113116--1264045--upload_2023-6-29_19-29-25.png2064×937 49.4 KB
9113116--1264048--upload_2023-6-29_19-29-34.png
9113116--1264048--upload_2023-6-29_19-29-34.png1845×923 151 KB

6

Ok, please keep me updated on the results of any experiments you try. There is definitely a problem here - this code should never crash. But I’m not yet sure what could cause it.

7

2022.3.4 didn’t help, apart from just removing that if statement at line 89 in Image.cpp (which I have no idea what that will do) im out of ideas

8

I think I’m actually seeing the same issue. Upgrading to 2022.3.4 also didn’t help. I think I’ll try downgrading back to 2021 and see if that resolves it.

9

Which if statement is that? Is it this one?

if (nhdr->n_type == NT_GNU_BUILD_ID)
10

Yes that’s the line sentry is reporting as the issue, I haven’t actually removed it and tested it, I was just assuming that’s something I could do

11

Ahh, ok, I see that in the screen shot now. I don’t think that can be removed. But you could add another check there. Maybe nhdr is NULL for some reason.

Can you try changing that line to this?

if (nhdr != NULL && nhdr->n_type == NT_GNU_BUILD_ID)

I wonder if that will mitigate the crashes.

12

Didn’t make a difference, still getting them come through

13

I’m not sure where to look then to find the cause.

Even though you cannot reproduce this locally, can you submit a bug report? Maybe we can try different devices or some other stress test workflow to reproduce it.

14

My current fix for this is to not allow exceptions to be raised so make sure the game is bug free :confused:

15

Ok, if you happen to notice that this is reproducible, please respond here again. I’d love to understand what is happening.

16

I sent a report in but I’m not able to attach my project and currently can’t build a reproducible #46678, i just attached an apk.

17

Ok we will see what we can do. We may need a way to reproduce this one.

18

What is the fault addr of these crashes?
I saw the crash in the first post happened on an ARMv7 device. Did the crash also happen on ARM64 devices, or it happens on ARMv7 only?

19

Looking into the implementation of GetELFImageBuildID, multiple Elf_Notes are considered to be contiguous:

void GetELFImageBuildID(char* build_id)
{
    // ...
 
    size_t nhdr_ptr = phdr[i].p_offset + imageBase;
    int j = 0;
    while (nhdr_ptr < imageBase + phdr[i].p_offset + phdr[i].p_memsz)
    {
        ElfW(Nhdr) * nhdr = (ElfW(Nhdr) *)nhdr_ptr;
        if (nhdr->n_type == NT_GNU_BUILD_ID)
        {
            char* image_build_id = (char *)((size_t)nhdr + sizeof(ElfW(Nhdr)) + nhdr->n_namesz);
            for (int j = 0; j < nhdr->n_descsz; j++)
            {
                snprintf(&build_id[j * 2], 3, "%02x", image_build_id[j]);
            }
            return;
        }
        nhdr_ptr += sizeof(ElfW(Nhdr)) + nhdr->n_descsz + nhdr->n_namesz;
    }
}

But according to the comment of Elf_Note and a documentation, there may be some padding bytes between them:

/*
 * Note header.  The ".note" section contains an array of notes.  Each
 * begins with this header, aligned to a word boundary.  Immediately
 * following the note header is n_namesz bytes of name, padded to the
 * next word boundary.  Then comes n_descsz bytes of descriptor, again
 * padded to a word boundary.  The values of n_namesz and n_descsz do
 * not include the padding.
 */
typedef struct {
    uint32_t    n_namesz;    /* Length of name. */
    uint32_t    n_descsz;    /* Length of descriptor. */
    uint32_t    n_type;        /* Type of this note. */
} Elf_Note;

If n_namesz or n_descsz happens to need some padding bytes, nhdr may become an address that’s not aligned to 4 bytes. When running nhdr->n_type on ARMv7 devices, application will crash due to unaligned memory access.

This is just my speculation, hope it’s helpful to investigate the crash further.

UPDATE:
Based on the speculation above, I created a minimal repro project: IN-49426. It can reproduce the following crash:

1 Like
20

Thank you for those details! And thanks for the case to reproduce this. I’ve asked our QA team to collapse these two bug reports IN-49426 and IN-46678 into one report. Then we can correct this.

2 Likes