183329 2018-03-05 06:44:40 -0800 Define GIGACAGE_ALLOCATION_CAN_FAIL on Linux 2022-11-23 08:59:25 -0800 1 1 1 Unclassified WebKit bmalloc WebKit Nightly Build PC Linux RESOLVED FIXED https://launchpad.net/bugs/1751460 https://bugzilla.redhat.com/show_bug.cgi?id=1554873 https://bugs.webkit.org/show_bug.cgi?id=248267 Gtk, InRadar P3 Normal --- 1 jbicha mcatanzaro bugs-noreply cgarcia commit-queue dytynenko.roman fpizlo fweimer ggaren kondor.dani mcatanzaro tpopela webkit-bug-importer ysuzuki oldest_to_newest 1403608 0 jbicha 2018-03-05 06:44:40 -0800 Ubuntu 18.04 (pre-Beta) recently upgraded to webkit2gtk 2.19.91. We are receiving lots of crash reports from Deja Dup at https://launchpad.net/bugs/1751460 Deja Dup is versioned 37.1. Stacktrace ---------- #0 0x00007fac7224d588 in () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #1 0x00007fac76c64827 in __pthread_once_slow (once_control=0x7fac724b502c, init_routine=0x7fac69330490 <__once_proxy>) at pthread_once.c:116 _buffer = {__routine = 0x7fac76c64880 <clear_once_control>, __arg = 0x7fac724b502c, __canceltype = 2008932720, __prev = 0x0} val = <optimized out> newval = <optimized out> #2 0x00007fac7224ce0d in Gigacage::ensureGigacage() () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #3 0x00007fac7224dd01 in bmalloc::Heap::Heap(bmalloc::HeapKind, std::lock_guard<bmalloc::StaticMutex>&) () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #4 0x00007fac7224bb10 in bmalloc::PerProcess<bmalloc::PerHeapKind<bmalloc::Heap> >::getSlowCase() () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #5 0x00007fac7224b7d4 in bmalloc::Cache::Cache(bmalloc::HeapKind) () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #6 0x00007fac7224bbf6 in bmalloc::PerThread<bmalloc::PerHeapKind<bmalloc::Cache> >::getSlowCase() () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #7 0x00007fac7224b87f in bmalloc::Cache::allocateSlowCaseNullCache(bmalloc::HeapKind, unsigned long) () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #8 0x00007fac72230f56 in WTF::StringImpl::createFromLiteral(char const*, unsigned int) () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #9 0x00007fac72230fe1 in WTF::StringImpl::createFromLiteral(char const*) () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #10 0x00007fac7223d3c0 in WTF::String::String(WTF::ASCIILiteral) () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #11 0x00007fac729f7557 in () at /usr/lib/x86_64-linux-gnu/libwebkit2gtk-4.0.so.37 #12 0x00007fac779c5733 in call_init (env=0x7ffc3677ae08, argv=0x7ffc3677adf8, argc=1, l=<optimized out>) at dl-init.c:72 j = <optimized out> jm = <optimized out> addrs = <optimized out> init_array = <optimized out> env = 0x7ffc3677ae08 argv = 0x7ffc3677adf8 argc = 1 l = <optimized out> preinit_array = <optimized out> preinit_array_size = <optimized out> i = 6 #13 0x00007fac779c5733 in _dl_init (main_map=0x7fac77bde170, argc=1, argv=0x7ffc3677adf8, env=0x7ffc3677ae08) at dl-init.c:119 preinit_array = <optimized out> preinit_array_size = <optimized out> i = 6 #14 0x00007fac779b60ca in _dl_start_user () at /lib64/ld-linux-x86-64.so.2 #15 0x0000000000000001 in () #16 0x00007ffc3677b9c3 in () #17 0x0000000000000000 in () 1403610 1 jbicha 2018-03-05 06:45:36 -0800 Thread Stacktrace ================= . Thread 1 (Thread 0x7fac77b6aac0 (LWP 3178)): #0 0x00007fac7224d588 in () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #1 0x00007fac76c64827 in __pthread_once_slow (once_control=0x7fac724b502c, init_routine=0x7fac69330490 <__once_proxy>) at pthread_once.c:116 _buffer = {__routine = 0x7fac76c64880 <clear_once_control>, __arg = 0x7fac724b502c, __canceltype = 2008932720, __prev = 0x0} val = <optimized out> newval = <optimized out> #2 0x00007fac7224ce0d in Gigacage::ensureGigacage() () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #3 0x00007fac7224dd01 in bmalloc::Heap::Heap(bmalloc::HeapKind, std::lock_guard<bmalloc::StaticMutex>&) () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #4 0x00007fac7224bb10 in bmalloc::PerProcess<bmalloc::PerHeapKind<bmalloc::Heap> >::getSlowCase() () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #5 0x00007fac7224b7d4 in bmalloc::Cache::Cache(bmalloc::HeapKind) () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #6 0x00007fac7224bbf6 in bmalloc::PerThread<bmalloc::PerHeapKind<bmalloc::Cache> >::getSlowCase() () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #7 0x00007fac7224b87f in bmalloc::Cache::allocateSlowCaseNullCache(bmalloc::HeapKind, unsigned long) () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #8 0x00007fac72230f56 in WTF::StringImpl::createFromLiteral(char const*, unsigned int) () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #9 0x00007fac72230fe1 in WTF::StringImpl::createFromLiteral(char const*) () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #10 0x00007fac7223d3c0 in WTF::String::String(WTF::ASCIILiteral) () at /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #11 0x00007fac729f7557 in () at /usr/lib/x86_64-linux-gnu/libwebkit2gtk-4.0.so.37 #12 0x00007fac779c5733 in call_init (env=0x7ffc3677ae08, argv=0x7ffc3677adf8, argc=1, l=<optimized out>) at dl-init.c:72 j = <optimized out> jm = <optimized out> addrs = <optimized out> init_array = <optimized out> env = 0x7ffc3677ae08 argv = 0x7ffc3677adf8 argc = 1 l = <optimized out> preinit_array = <optimized out> preinit_array_size = <optimized out> i = 6 #13 0x00007fac779c5733 in _dl_init (main_map=0x7fac77bde170, argc=1, argv=0x7ffc3677adf8, env=0x7ffc3677ae08) at dl-init.c:119 preinit_array = <optimized out> preinit_array_size = <optimized out> i = 6 #14 0x00007fac779b60ca in _dl_start_user () at /lib64/ld-linux-x86-64.so.2 #15 0x0000000000000001 in () #16 0x00007ffc3677b9c3 in () #17 0x0000000000000000 in () 1403614 2 cgarcia 2018-03-05 06:51:05 -0800 The bt looks similar to the one in https://bugs.webkit.org/show_bug.cgi?id=179914#c39 1403628 3 mcatanzaro 2018-03-05 09:08:26 -0800 The failure occurs here: // FIXME: Randomize where this goes. // https://bugs.webkit.org/show_bug.cgi?id=175245 void* base = tryVMAllocate(maxAlignment, totalSize); if (!base) { if (GIGACAGE_ALLOCATION_CAN_FAIL) return; fprintf(stderr, "FATAL: Could not allocate gigacage memory with maxAlignment = %lu, totalSize = %lu.\n", maxAlignment, totalSize); BCRASH(); } So tryVMAllocate fails. That means bmalloc was unable to allocate virtual memory. That's not supposed to fail (obviously). Implementation is here: inline void* tryVMAllocate(size_t vmSize) { vmValidate(vmSize); void* result = mmap(0, vmSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | BMALLOC_NORESERVE, BMALLOC_VM_TAG, 0); if (result == MAP_FAILED) return nullptr; return result; } So the problem boils down to this mmap call. It's very strange that this is only happening with Deja Dup. Other applications are unaffected? 1403629 4 mcatanzaro 2018-03-05 09:13:36 -0800 Jeremy, here's some debug you could try adding to Source/bmalloc/bmalloc/VMAllocate.h: // At the top of the file, before the bmalloc namespace #include <cstring> #include <errno.h> inline void* tryVMAllocate(size_t vmSize) { vmValidate(vmSize); void* result = mmap(0, vmSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | BMALLOC_NORESERVE, BMALLOC_VM_TAG, 0); if (result == MAP_FAILED) { WTFLogAlways("%s: mmap failed: %d (%s)", __FUNCTION__, errno, strerror(errno)); return nullptr; } return result; } That would tell us which of the many possible errors are occurring here. And if you need an immediate workaround, you can of course build with -DUSE_SYSTEM_MALLOC=ON. That will be bad, so I can't recommend that... but you're already disabling GStreamerGL and web fonts.... :P 1403633 5 mcatanzaro 2018-03-05 09:15:35 -0800 (In reply to Michael Catanzaro from comment #3) > fprintf(stderr, "FATAL: Could not allocate gigacage memory > with maxAlignment = %lu, totalSize = %lu.\n", maxAlignment, totalSize); This might be helpful too. That should also print before the crash. (In reply to Michael Catanzaro from comment #4) > inline void* tryVMAllocate(size_t vmSize) > { > vmValidate(vmSize); > void* result = mmap(0, vmSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | > MAP_ANON | BMALLOC_NORESERVE, BMALLOC_VM_TAG, 0); > if (result == MAP_FAILED) > { > WTFLogAlways("%s: mmap failed: %d (%s)", __FUNCTION__, errno, > strerror(errno)); > return nullptr; > } > return result; > } I forgot we can't use WTF here. Got to use fprintf: fprintf(stderr, "%s: mmap failed: %d (%s)", __FUNCTION__, errno, strerror(errno)); 1403634 6 ysuzuki 2018-03-05 09:22:47 -0800 (In reply to Michael Catanzaro from comment #4) > Jeremy, here's some debug you could try adding to > Source/bmalloc/bmalloc/VMAllocate.h: > > // At the top of the file, before the bmalloc namespace > #include <cstring> > #include <errno.h> > > inline void* tryVMAllocate(size_t vmSize) > { > vmValidate(vmSize); > void* result = mmap(0, vmSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | > MAP_ANON | BMALLOC_NORESERVE, BMALLOC_VM_TAG, 0); > if (result == MAP_FAILED) > { > WTFLogAlways("%s: mmap failed: %d (%s)", __FUNCTION__, errno, > strerror(errno)); > return nullptr; > } > return result; > } > > That would tell us which of the many possible errors are occurring here. > > And if you need an immediate workaround, you can of course build with > -DUSE_SYSTEM_MALLOC=ON. That will be bad, so I can't recommend that... but > you're already disabling GStreamerGL and web fonts.... :P The immediate fix is disabling Gigacage by setting GIGACAGE_ENABLED 0 in bmalloc/Gigacage.h. This keeps bmalloc, but disables Gigacage. My guess is that Linux fails to mmap regions and returns MAP_FAILED if the size is very large. But I'm not sure right now since it is working on my environment... Anyway, @mcatanzaro, do you know the way to allocate virtual memory region which does not have actual backing pages? 1403646 7 ysuzuki 2018-03-05 09:48:33 -0800 (In reply to Yusuke Suzuki from comment #6) > (In reply to Michael Catanzaro from comment #4) > > Jeremy, here's some debug you could try adding to > > Source/bmalloc/bmalloc/VMAllocate.h: > > > > // At the top of the file, before the bmalloc namespace > > #include <cstring> > > #include <errno.h> > > > > inline void* tryVMAllocate(size_t vmSize) > > { > > vmValidate(vmSize); > > void* result = mmap(0, vmSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | > > MAP_ANON | BMALLOC_NORESERVE, BMALLOC_VM_TAG, 0); > > if (result == MAP_FAILED) > > { > > WTFLogAlways("%s: mmap failed: %d (%s)", __FUNCTION__, errno, > > strerror(errno)); > > return nullptr; > > } > > return result; > > } > > > > That would tell us which of the many possible errors are occurring here. > > > > And if you need an immediate workaround, you can of course build with > > -DUSE_SYSTEM_MALLOC=ON. That will be bad, so I can't recommend that... but > > you're already disabling GStreamerGL and web fonts.... :P > > The immediate fix is disabling Gigacage by setting GIGACAGE_ENABLED 0 in > bmalloc/Gigacage.h. > This keeps bmalloc, but disables Gigacage. > > My guess is that Linux fails to mmap regions and returns MAP_FAILED if the > size is very large. > But I'm not sure right now since it is working on my environment... > Anyway, @mcatanzaro, do you know the way to allocate virtual memory region > which does not have actual backing pages? My guess is that mmap with READ/WRITE starts populating backing pages and it fails. I think we potentially have two ways to fix this issue. The first way is, 1. first allocate region with mmap NO_PROTO 2. set read/write if necessary But since Darwin is not requiring this mechanism, I think we need to change the current code a bit. For example, vmAllocate() assumes that returned memory is accessible. But this assumption is slightly broken in Linux if we take this way. We need a bit additional code for Linux. The second way is 1. first allocate region with mmap NO_PROTO 2. deallocate backing pages 3. set read/write for this region immediately I'm not sure it works. But if it works, the change should be minimum. 1403651 8 mcatanzaro 2018-03-05 09:55:48 -0800 One thing I just noticed, from https://bugzilla.gnome.org/show_bug.cgi?id=794056, is that this stack trace is exactly the same as the one valgrind hits when trying to use valgrind to debug epiphany since gigacage was enabled. That's probably expected, since valgrind attempts to allocate shadow bytes to keep track of the entire address space, which isn't practical under gigacage. (In reply to Yusuke Suzuki from comment #6) > Anyway, @mcatanzaro, do you know the way to allocate virtual memory region > which does not have actual backing pages? I have no clue about this. I thought actual backing pages were only employed once the memory region is actually used. 1403652 9 mcatanzaro 2018-03-05 10:00:35 -0800 (In reply to Yusuke Suzuki from comment #6) > My guess is that Linux fails to mmap regions and returns MAP_FAILED if the > size is very large. Keep in mind, we've had gigacage enabled and working fine for half a year.... 1403653 10 ysuzuki 2018-03-05 10:03:36 -0800 (In reply to Michael Catanzaro from comment #8) > One thing I just noticed, from > https://bugzilla.gnome.org/show_bug.cgi?id=794056, is that this stack trace > is exactly the same as the one valgrind hits when trying to use valgrind to > debug epiphany since gigacage was enabled. That's probably expected, since > valgrind attempts to allocate shadow bytes to keep track of the entire > address space, which isn't practical under gigacage. It sounds reasonable assumption to me. > > (In reply to Yusuke Suzuki from comment #6) > > Anyway, @mcatanzaro, do you know the way to allocate virtual memory region > > which does not have actual backing pages? > > I have no clue about this. I thought actual backing pages were only employed > once the memory region is actually used. (In reply to Michael Catanzaro from comment #9) > (In reply to Yusuke Suzuki from comment #6) > > My guess is that Linux fails to mmap regions and returns MAP_FAILED if the > > size is very large. > > Keep in mind, we've had gigacage enabled and working fine for half a year.... Yeah, right. If it returns MAP_FAILED due to this issue, we should more constantly see this issue in our daily development. So my guess seems wrong. 1403672 11 mcatanzaro 2018-03-05 10:37:00 -0800 Something could be different in Ubuntu, or in Deja Dup specifically? I'm actually really surprised that Deja Dup now depends on WebKit. Must be via gnome-online-accounts. 1403673 12 jbicha 2018-03-05 10:39:54 -0800 (In reply to Michael Catanzaro from comment #11) > I'm actually really surprised that Deja Dup now depends on WebKit. Must be > via gnome-online-accounts. Good question. I don't see any webkit code in Deja Dup but it does use GOA. 1406115 13 mcatanzaro 2018-03-13 09:00:09 -0700 *** Bug 183594 has been marked as a duplicate of this bug. *** 1406120 14 mcatanzaro 2018-03-13 09:25:25 -0700 So now we have: * Our first bug reports against applications other than Deja Dup * Our first bug report from a distro other than Ubuntu Seems like a serious problem. 1406121 15 mcatanzaro 2018-03-13 09:26:39 -0700 (In reply to Michael Catanzaro from comment #4) > Jeremy, here's some debug you could try adding to > Source/bmalloc/bmalloc/VMAllocate.h ^ This remains the next step here. We need to know why the mmap() call is failing. 1406307 16 jbicha 2018-03-13 17:25:51 -0700 (In reply to Michael Catanzaro from comment #4) > Jeremy, here's some debug you could try adding to > Source/bmalloc/bmalloc/VMAllocate.h: webkit2gtk failed to build for me with those lines. https://launchpad.net/~jbicha/+archive/ubuntu/tempdeja/+sourcepub/8853741/+listing-archive-extra Build log excerpt ================= In file included from /<<PKGBUILDDIR>>/Source/bmalloc/bmalloc/SmallPage.h:32:0, from /<<PKGBUILDDIR>>/Source/bmalloc/bmalloc/Deallocator.h:31, from /<<PKGBUILDDIR>>/Source/bmalloc/bmalloc/Cache.h:31, from /<<PKGBUILDDIR>>/Source/bmalloc/bmalloc/bmalloc.h:29, from /<<PKGBUILDDIR>>/Source/bmalloc/bmalloc/BMalloced.h:28, from /<<PKGBUILDDIR>>/Source/bmalloc/bmalloc/IsoHeapImpl.h:28, from /<<PKGBUILDDIR>>/Source/bmalloc/bmalloc/AllIsoHeaps.h:28, from /<<PKGBUILDDIR>>/Source/bmalloc/bmalloc/AllIsoHeaps.cpp:26: /<<PKGBUILDDIR>>/Source/bmalloc/bmalloc/VMAllocate.h: In function ‘void* tryVMAllocate(size_t)’: /<<PKGBUILDDIR>>/Source/bmalloc/bmalloc/VMAllocate.h:48:5: error: ‘vmValidate’ was not declared in this scope vmValidate(vmSize); ^~~~~~~~~~ /<<PKGBUILDDIR>>/Source/bmalloc/bmalloc/VMAllocate.h:49:85: error: ‘BMALLOC_NORESERVE’ was not declared in this scope void* result = mmap(0, vmSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | BMALLOC_NORESERVE, BMALLOC_VM_TAG, 0); ^~~~~~~~~~~~~~~~~ /<<PKGBUILDDIR>>/Source/bmalloc/bmalloc/VMAllocate.h:49:104: error: ‘BMALLOC_VM_TAG’ was not declared in this scope void* result = mmap(0, vmSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | BMALLOC_NORESERVE, BMALLOC_VM_TAG, 0); 1406309 17 mcatanzaro 2018-03-13 17:37:08 -0700 (In reply to Jeremy Bicha from comment #16) > (In reply to Michael Catanzaro from comment #4) > > Jeremy, here's some debug you could try adding to > > Source/bmalloc/bmalloc/VMAllocate.h: > > webkit2gtk failed to build for me with those lines. Do: keep the two new #includes where you added them, at the top, above the bmalloc namespace Don't: add a new definition of tryVMAllocate. Just add the print to the one that already exists, lower in the file. Watch the braces. 1406341 18 mcatanzaro 2018-03-13 19:58:02 -0700 Here's a debug patch: diff --git a/Source/bmalloc/bmalloc/VMAllocate.h b/Source/bmalloc/bmalloc/VMAllocate.h index 713e9c9fbb6..5c9c8553b1a 100644 --- a/Source/bmalloc/bmalloc/VMAllocate.h +++ b/Source/bmalloc/bmalloc/VMAllocate.h @@ -35,6 +35,9 @@ #include <sys/mman.h> #include <unistd.h> +#include <cstdio> +#include <errno.h> + #if BOS(DARWIN) #include <mach/vm_page_size.h> #include <mach/vm_statistics.h> @@ -123,7 +126,10 @@ inline void* tryVMAllocate(size_t vmSize) vmValidate(vmSize); void* result = mmap(0, vmSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | BMALLOC_NORESERVE, BMALLOC_VM_TAG, 0); if (result == MAP_FAILED) +{ +fprintf(stderr, "%s: mmap failed: vmSize=%zu errno=%d (%s)", __FUNCTION__, vmSize, errno, strerror(errno)); return nullptr; +} return result; } 1406365 19 tpopela 2018-03-14 01:46:26 -0700 (In reply to Michael Catanzaro from comment #18) > Here's a debug patch: It's missing <cstring> include for strerror.. diff -up webkitgtk-2.20.0/Source/bmalloc/bmalloc/VMAllocate.h.bmalloc_debug webkitgtk-2.20.0/Source/bmalloc/bmalloc/VMAllocate.h --- webkitgtk-2.20.0/Source/bmalloc/bmalloc/VMAllocate.h.bmalloc_debug 2018-02-19 08:45:33.000000000 +0100 +++ webkitgtk-2.20.0/Source/bmalloc/bmalloc/VMAllocate.h 2018-03-14 09:26:14.977674938 +0100 @@ -35,6 +35,10 @@ #include <sys/mman.h> #include <unistd.h> +#include <cstdio> +#include <errno.h> +#include <cstring> + #if BOS(DARWIN) #include <mach/vm_page_size.h> #include <mach/vm_statistics.h> @@ -122,8 +126,10 @@ inline void* tryVMAllocate(size_t vmSize { vmValidate(vmSize); void* result = mmap(0, vmSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | BMALLOC_NORESERVE, BMALLOC_VM_TAG, 0); - if (result == MAP_FAILED) - return nullptr; + if (result == MAP_FAILED) { + fprintf(stderr, "%s: mmap failed: vmSize=%zu errno=%d (%s)", __FUNCTION__, vmSize, errno, strerror(errno)); + return nullptr; + } return result; } 1406372 20 mcatanzaro 2018-03-14 06:47:41 -0700 I swear I tested it, but I tested on trunk, so maybe cstring was getting pulled in elsewhere. 1406373 21 tpopela 2018-03-14 06:53:14 -0700 (In reply to Michael Catanzaro from comment #20) > I swear I tested it, but I tested on trunk, so maybe cstring was getting > pulled in elsewhere. Nevermind, see Jiri's response in https://bugzilla.redhat.com/show_bug.cgi?id=1554873#c6. Updating mutter fixed the issue.. 1406375 22 mcatanzaro 2018-03-14 07:04:32 -0700 (In reply to Michael Catanzaro from comment #15) > ^ This remains the next step here. We need to know why the mmap() call is > failing. Jeremy discovered Deja Dup is setting a virtual memory limit in its desktop file: Exec=sh -c "ulimit -v 1000000; exec @pkglibexecdir@/deja-dup-monitor" which is incompatible with Gigacage. It needs to not do that. This is NOTABUG. Good find, Jeremy! (In reply to Tomas Popela from comment #21) > Nevermind, see Jiri's response in > https://bugzilla.redhat.com/show_bug.cgi?id=1554873#c6. Updating mutter > fixed the issue.. The mutter update would definitely fix the problem described (app fails to launch another app), but I don't see how it could possibly be related to the ensureGigacage() backtrace. Anyway, that bug is closed now too... we can always revisit if it becomes a problem again. 1406376 23 jbicha 2018-03-14 07:11:31 -0700 To give some more context about what Deja Dup was doing… See https://salsa.debian.org/gnome-team/deja-dup/blob/debian/master/data/org.gnome.DejaDup.Monitor.desktop.in which points to https://launchpad.net/bugs/1302416 And here's the output when I ran the command via webkit2gtk 2.20 built with the debugging patch proposed earlier here $ ulimit -v 1000000; /usr/lib/deja-dup/deja-dup-monitor tryVMAllocate: mmap failed: vmSize=154618822656 errno=12 (Cannot allocate memory) FATAL: Could not allocate gigacage memory with maxAlignment = 34359738368, totalSize = 120259084288. Segmentation fault (core dumped) 1406382 24 mcatanzaro 2018-03-14 07:32:02 -0700 Seb from Ubuntu is worried that people will have set a vmlimit for the entire desktop session, and requests that we improve the error message here. 1406385 25 335768 mcatanzaro 2018-03-14 07:41:23 -0700 Created attachment 335768 Patch 1406386 26 tpopela 2018-03-14 07:43:23 -0700 (In reply to Michael Catanzaro from comment #25) > Created attachment 335768 [details] > Patch Informal r+ as we spoke together on IRC.. 1406409 27 mcatanzaro 2018-03-14 09:06:59 -0700 Filip, Geoffrey, does this error message look OK to you? Want any adjustments? 1406437 28 dytynenko.roman 2018-03-14 10:39:47 -0700 ulimit -a returned virtual memory unlimited, but I figured out that this error was triggered because I explicitly disabled overcommit by vm.overcommit_memory = 2 setting it to 1 fixed the issue 1406444 29 335768 fpizlo 2018-03-14 10:53:06 -0700 Comment on attachment 335768 Patch You might want to set GIGACAEG_ALLOCATION_CAN_FAIL to true on Linux until that's fixed. 1406471 30 mcatanzaro 2018-03-14 12:46:10 -0700 (In reply to dytynenko.roman from comment #28) > ulimit -a returned virtual memory unlimited, but I figured out that this > error was triggered because I explicitly disabled overcommit by > vm.overcommit_memory = 2 > setting it to 1 fixed the issue Oh boy... (In reply to Filip Pizlo from comment #29) > Comment on attachment 335768 [details] > Patch > > You might want to set GIGACAEG_ALLOCATION_CAN_FAIL to true on Linux until > that's fixed. I'm not sure what we should do here. We definitely don't need to change anything for Deja Dup; it's easier to fix Deja Dup than to update WebKit, anyway. Disabling overcommit is worth considering, though, and the connection between overcommit and browser security is not obvious. * If we leave things unchanged, WebKit will crash with overcommit disabled. * If we set GIGACAGE_ALLOCATION_CAN_FAIL, WebKit will not crash, but an important security feature will be lost, and users will not notice. My inclination is to say Gigacage is sufficiently important that WebKit should crash if overcommit is disabled. Perhaps that's not very kind, though. Thoughts? 1407004 31 335768 mcatanzaro 2018-03-16 09:41:04 -0700 Comment on attachment 335768 Patch Let's keep this error message. System administrators should consider the needs of the software they are running before setting virtual memory limits or disabling overcommit. 1407020 32 335768 commit-queue 2018-03-16 10:06:07 -0700 Comment on attachment 335768 Patch Clearing flags on attachment: 335768 Committed r229669: <https://trac.webkit.org/changeset/229669> 1407021 33 commit-queue 2018-03-16 10:06:09 -0700 All reviewed patches have been landed. Closing bug. 1407022 34 webkit-bug-importer 2018-03-16 10:07:18 -0700 <rdar://problem/38548430> 1407049 35 ysuzuki 2018-03-16 10:57:40 -0700 (In reply to dytynenko.roman from comment #28) > ulimit -a returned virtual memory unlimited, but I figured out that this > error was triggered because I explicitly disabled overcommit by > vm.overcommit_memory = 2 > setting it to 1 fixed the issue OVERCOMMIT_NEVER (2) makes mmap's flag MAP_NORESERVE meaningless. It makes Gigacage address space allocation failed. https://github.com/torvalds/linux/blob/master/mm/mmap.c#L1475 So it sounds reasonable to me that vm.overcommit_memory=0 or 1 fixes this issue. 1416832 36 tpopela 2018-04-23 01:32:41 -0700 So it looks like there is a way of avoiding this in most cases. Florian Weimer suggested the following in https://bugzilla.redhat.com/show_bug.cgi?id=1570021: mmap(NULL, 154618822656, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0) = -1 ENOMEM (Cannot allocate memory) MAP_NORESERVE does not have the intended effect with vm.overcommit_memory=2 (which enables the beancounter/disables overcommit). The standard way of reserving address space is to use a PROT_NONE mapping and make parts readable/writable as needed using mprotect later. This will work with vm.overcommit_memory=2 as well because initially PROT_NONE mappings do not count towards the commit limit. Yusuke, would you mind looking at this? You seem to know this low level stuff best. 1416853 37 mcatanzaro 2018-04-23 05:59:29 -0700 Of course he notices due to Emacs... OK, reopening. 1422632 38 kondor.dani 2018-05-09 22:33:38 -0700 Hi, I just started having problems due to this issue recently on Ubuntu 16.04.4 with libjavascriptcoregtk-4.0-18 version 2.20.1-0ubuntu0.16.04.1. Several Gnome 3.18 applications will crash on startup (including Rhythmbox 3.3, yelp, gnome-control-center, unity-control-center 15.04.0+16.04.20171130-0ubuntu1, zenity, but basically anything that depends on WebKit I guess). I get the following (or similar) error message: FATAL: Could not allocate gigacage memory with maxAlignment = 34359738368, totalSize = 103079215104. (Make sure you have not set a virtual memory limit.) Setting the environment variable GIGACAGE_ENABLED=0 is a good workaround. I have overcommit_memory = 2, and I prefer to have it that way (short reason: I develop and run various programs for scientific computations, which sometimes need a lot of memory -- in my case, it is clearly preferable to have a memory allocation fail, producing a meaningful error message than having a seemingly random crash later or invoking the OOM killer). Nevertheless, I still use Gnome desktop applications, which depend on WebKit, producing this issue. My understanding is that the only way to fix this is to use PROT_NONE when allocating address space and later use mprotect() before actually using it. I understand that depending the structure of the code, that could be a lot of work to do for a somewhat specialized use case. Let me know if I can provide any more info. Stack trace: Thread 1 (Thread 0x7ffff7edda80 (LWP 16799)): #0 0x00007fffeccdb905 in ?? () from /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #1 0x00007ffff66b4a99 in __pthread_once_slow (once_control=0x7fffecf49008, init_routine=0x7fffe8aceac0 <__once_proxy>) at pthread_once.c:116 #2 0x00007fffeccdb243 in Gigacage::ensureGigacage() () from /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #3 0x00007fffecce05a8 in bmalloc::mapToActiveHeapKind(bmalloc::HeapKind) () from /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #4 0x00007fffeccd9d3e in bmalloc::Cache::allocateSlowCaseNullCache(bmalloc::HeapKind, unsigned long) () from /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #5 0x00007fffeccbf6f6 in WTF::StringImpl::createFromLiteral(char const*, unsigned int) () from /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #6 0x00007fffeccbf781 in WTF::StringImpl::createFromLiteral(char const*) () from /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #7 0x00007fffecccbc20 in WTF::String::String(WTF::ASCIILiteral) () from /usr/lib/x86_64-linux-gnu/libjavascriptcoregtk-4.0.so.18 #8 0x00007ffff3865a17 in ?? () from /usr/lib/x86_64-linux-gnu/libwebkit2gtk-4.0.so.37 #9 0x00007ffff7de76ba in call_init (l=<optimized out>, argc=argc@entry=1, argv=argv@entry=0x7fffffffe608, env=env@entry=0x7fffffffe618) at dl-init.c:72 #10 0x00007ffff7de77cb in call_init (env=0x7fffffffe618, argv=0x7fffffffe608, argc=1, l=<optimized out>) at dl-init.c:30 #11 _dl_init (main_map=0x7ffff7ffe168, argc=1, argv=0x7fffffffe608, env=0x7fffffffe618) at dl-init.c:120 #12 0x00007ffff7dd7c6a in _dl_start_user () from /lib64/ld-linux-x86-64.so.2 #13 0x0000000000000001 in ?? () #14 0x00007fffffffe91c in ?? () #15 0x0000000000000000 in ?? () 1422710 39 mcatanzaro 2018-05-10 07:14:24 -0700 (In reply to Daniel Kondor from comment #38) > My understanding is that the only way to fix this is to use PROT_NONE when > allocating address space and later use mprotect() before actually using it. > I understand that depending the structure of the code, that could be a lot > of work to do for a somewhat specialized use case. Yeah, that's exactly what needs to happen here. I took a look at this, but wasn't sure where the mprotect()s were needed. 1422711 40 ysuzuki 2018-05-10 07:15:27 -0700 (In reply to Michael Catanzaro from comment #39) > (In reply to Daniel Kondor from comment #38) > > My understanding is that the only way to fix this is to use PROT_NONE when > > allocating address space and later use mprotect() before actually using it. > > I understand that depending the structure of the code, that could be a lot > > of work to do for a somewhat specialized use case. > > Yeah, that's exactly what needs to happen here. I took a look at this, but > wasn't sure where the mprotect()s were needed. I'm a bit worried about this approach in terms of performance... 1423097 41 kondor.dani 2018-05-11 03:01:45 -0700 Just another thought on a potential further implication: if PROT_NONE was the default and mprotect() used to make individual pages writable / readable, any out of bounds accesses (due to e.g. any bugs / exploits that might come up later) have some chance of triggering a segfault if it falls to another page not reserved with mprotect(). Not being familiar with the code, I'm not sure if this would be a feature or if guard pages like this are already implemented. Just to point out that any change will probably have more implications than I originally would have thought :) 1425637 42 340850 ysuzuki 2018-05-21 06:30:57 -0700 Created attachment 340850 Patch 1425647 43 340850 mcatanzaro 2018-05-21 08:11:31 -0700 Comment on attachment 340850 Patch Well I'm not sure if we should do this, because Gigacage is an important security feature. There are some notes at https://labs.mwrinfosecurity.com/blog/some-brief-notes-on-webkit-heap-hardening/ about all the hardening features that this will silently disable. But I guess not crashing is good, so r=me. Did you try Florian's suggestion? Were there problems with performance? 1426089 44 ysuzuki 2018-05-22 03:45:55 -0700 (In reply to Michael Catanzaro from comment #43) > Comment on attachment 340850 [details] > Patch > > Well I'm not sure if we should do this, because Gigacage is an important > security feature. There are some notes at > https://labs.mwrinfosecurity.com/blog/some-brief-notes-on-webkit-heap- > hardening/ about all the hardening features that this will silently disable. > > But I guess not crashing is good, so r=me. > > Did you try Florian's suggestion? Were there problems with performance? Currently, I'm not trying this. I think currently this patch is OK since it avoids crashing. And users can enable gigacage by enabling overcommit :) 1426092 45 ysuzuki 2018-05-22 03:49:22 -0700 Committed r232059: <https://trac.webkit.org/changeset/232059> 335768 2018-03-14 07:41:23 -0700 2018-05-21 06:30:55 -0700 Patch bug-183329-20180314094122.patch text/plain 1980 mcatanzaro U3VidmVyc2lvbiBSZXZpc2lvbjogMjI5NTg2CmRpZmYgLS1naXQgYS9Tb3VyY2UvYm1hbGxvYy9D aGFuZ2VMb2cgYi9Tb3VyY2UvYm1hbGxvYy9DaGFuZ2VMb2cKaW5kZXggMTNjYWQ2MWZjMDBkOTM3 Nzc0NTcxZDAxNDc0MTc1ZGI0ODZiZWE2Yi4uOGYzZTQ3OTVlNzllOGFiNTkyMmIwMjgxMmIwNDVl MTg3YjcwMDM2OSAxMDA2NDQKLS0tIGEvU291cmNlL2JtYWxsb2MvQ2hhbmdlTG9nCisrKyBiL1Nv dXJjZS9ibWFsbG9jL0NoYW5nZUxvZwpAQCAtMSwzICsxLDIxIEBACisyMDE4LTAzLTE0ICBNaWNo YWVsIENhdGFuemFybyAgPG1jYXRhbnphcm9AaWdhbGlhLmNvbT4KKworICAgICAgICBJbXByb3Zl IGVycm9yIG1lc3NhZ2Ugd2hlbiBHaWdhY2FnZSBjYW5ub3QgYWxsb2NhdGUgdmlydHVhbCBtZW1v cnkKKyAgICAgICAgaHR0cHM6Ly9idWdzLndlYmtpdC5vcmcvc2hvd19idWcuY2dpP2lkPTE4MzMy OQorCisgICAgICAgIFJldmlld2VkIGJ5IE5PQk9EWSAoT09QUyEpLgorCisgICAgICAgIFdlJ3Zl IGRpc2NvdmVyZWQgdGhhdCBEZWphIER1cCBtb25pdG9yIHNldHMgYSB2aXJ0dWFsIG1lbW9yeSBs aW1pdCwgYnJlYWtpbmcgR2lnYWNhZ2UuIFNpbmNlCisgICAgICAgIGl0IHJ1bnMgaW4gdGhlIGJh Y2tncm91bmQgb24gYSBmcmVzaCBvdXQtb2YtdGhlLWJveCBpbnN0YWxsIG9mIFVidW50dSwgdGhp cyBpcyBub3QgZ29vZC4KKyAgICAgICAgVGhhdCB3aWxsIGhhdmUgdG8gYmUgZml4ZWQgYnkgRGVq YSBEdXAsIGJ1dCB0aGVyZSBpcyBjb25jZXJuIHRoYXQgb3RoZXIgYXBwbGljYXRpb25zIG1pZ2h0 CisgICAgICAgIHRyeSB0aGlzLCBvciB0aGF0IHVzZXJzIHdpbGwgc2V0IGEgdmlydHVhbCBtZW1v cnkgbGltaXQgZm9yIHRoZSBlbnRpcmUgZGVza3RvcCBzZXNzaW9uLiBPZgorICAgICAgICBwYXJ0 aWN1bGFyIGNvbmNlcm4gaXMgdGhlIHBvc3NpYmlsaXR5IHRoYXQgdXNlcnMgbWlnaHQgaGF2ZSBj b3B5cGFzdGVkIGEgdWxpbWl0IGxpbmUgaW50bworICAgICAgICBhIHNlc3Npb24gc3RhcnR1cCBz Y3JpcHQgd2l0aG91dCB1bmRlcnN0YW5kaW5nIGl0LiBMZXQncyB0cnkgdG8gbWFrZSBpdCBzbGln aHRseSBlYXNpZXIgdG8KKyAgICAgICAgdW5kZXJzdGFuZCB3aGF0J3MgZ29pbmcgd3JvbmcuCisK KyAgICAgICAgKiBibWFsbG9jL0dpZ2FjYWdlLmNwcDoKKyAgICAgICAgKEdpZ2FjYWdlOjplbnN1 cmVHaWdhY2FnZSk6CisKIDIwMTgtMDMtMTAgIEZpbGlwIFBpemxvICA8ZnBpemxvQGFwcGxlLmNv bT4KIAogICAgICAgICBQZXJQcm9jZXNzPD4gc2hvdWxkIGJlIHNhZmUgYnkgZGVmYXVsdApkaWZm IC0tZ2l0IGEvU291cmNlL2JtYWxsb2MvYm1hbGxvYy9HaWdhY2FnZS5jcHAgYi9Tb3VyY2UvYm1h bGxvYy9ibWFsbG9jL0dpZ2FjYWdlLmNwcAppbmRleCAwMDkwNjk5NjNhNGU3OTI1NTgzN2VlNGIx NWJlZjIzNzhjZTUwYmY2Li41ZTU2ZTJhZjdlMjY1YzUxYTk2Y2RlZmExZTY3MTRhNjViZTE5N2Vk IDEwMDY0NAotLS0gYS9Tb3VyY2UvYm1hbGxvYy9ibWFsbG9jL0dpZ2FjYWdlLmNwcAorKysgYi9T b3VyY2UvYm1hbGxvYy9ibWFsbG9jL0dpZ2FjYWdlLmNwcApAQCAtMTUxLDYgKzE1MSw3IEBAIHZv aWQgZW5zdXJlR2lnYWNhZ2UoKQogICAgICAgICAgICAgICAgIGlmIChHSUdBQ0FHRV9BTExPQ0FU SU9OX0NBTl9GQUlMKQogICAgICAgICAgICAgICAgICAgICByZXR1cm47CiAgICAgICAgICAgICAg ICAgZnByaW50ZihzdGRlcnIsICJGQVRBTDogQ291bGQgbm90IGFsbG9jYXRlIGdpZ2FjYWdlIG1l bW9yeSB3aXRoIG1heEFsaWdubWVudCA9ICVsdSwgdG90YWxTaXplID0gJWx1LlxuIiwgbWF4QWxp Z25tZW50LCB0b3RhbFNpemUpOworICAgICAgICAgICAgICAgIGZwcmludGYoc3RkZXJyLCAiKE1h a2Ugc3VyZSB5b3UgaGF2ZSBub3Qgc2V0IGEgdmlydHVhbCBtZW1vcnkgbGltaXQuKVxuIik7CiAg ICAgICAgICAgICAgICAgQkNSQVNIKCk7CiAgICAgICAgICAgICB9CiAK 340850 2018-05-21 06:30:57 -0700 2018-05-21 08:11:31 -0700 Patch bug-183329-20180521223056.patch text/plain 1579 ysuzuki U3VidmVyc2lvbiBSZXZpc2lvbjogMjMyMDE0CmRpZmYgLS1naXQgYS9Tb3VyY2UvYm1hbGxvYy9D aGFuZ2VMb2cgYi9Tb3VyY2UvYm1hbGxvYy9DaGFuZ2VMb2cKaW5kZXggODc4MjBhN2I2NzE1N2M5 NDYwNjkzYjQwNjJmZTk0ZmZhODM2ODNlYy4uZDVlMTQxZmIxN2E0YzFjNDYzOWUyZWYwOTNlMTQ1 MDE2NDgwN2NlMCAxMDA2NDQKLS0tIGEvU291cmNlL2JtYWxsb2MvQ2hhbmdlTG9nCisrKyBiL1Nv dXJjZS9ibWFsbG9jL0NoYW5nZUxvZwpAQCAtMSwzICsxLDE2IEBACisyMDE4LTA1LTIxICBZdXN1 a2UgU3V6dWtpICA8dXRhdGFuZS50ZWFAZ21haWwuY29tPgorCisgICAgICAgIEltcHJvdmUgaG93 IEdpZ2FjYWdlIHJlc2VydmVzIGFkZHJlc3Mgc3BhY2Ugb24gTGludXgKKyAgICAgICAgaHR0cHM6 Ly9idWdzLndlYmtpdC5vcmcvc2hvd19idWcuY2dpP2lkPTE4MzMyOQorCisgICAgICAgIFJldmll d2VkIGJ5IE5PQk9EWSAoT09QUyEpLgorCisgICAgICAgIFdlIHNwZWNpZnkgYEdJR0FDQUdFX0FM TE9DQVRJT05fQ0FOX0ZBSUwgMWAgaW4gTGludXggc2luY2UKKyAgICAgICAgTGludXggY2FuIGZh aWwgdG8gYG1tYXBgIGlmIGB2bS5vdmVyY29tbWl0X21lbW9yeSA9IDJgLgorICAgICAgICBVc2Vy cyBjYW4gZW5hYmxlIEdpZ2FjYWdlIGlmIHVzZXJzIGVuYWJsZSBvdmVyY29tbWl0X21lbW9yeS4K KworICAgICAgICAqIGJtYWxsb2MvR2lnYWNhZ2UuaDoKKwogMjAxOC0wNS0wNiAgWXVzdWtlIFN1 enVraSAgPHV0YXRhbmUudGVhQGdtYWlsLmNvbT4KIAogICAgICAgICBbSlNDXSBSZW1vdmUgInVz aW5nIG5hbWVzcGFjZSBzdGQ7IiBmcm9tIEpTQywgYm1hbGxvYywgV1RGCmRpZmYgLS1naXQgYS9T b3VyY2UvYm1hbGxvYy9ibWFsbG9jL0dpZ2FjYWdlLmggYi9Tb3VyY2UvYm1hbGxvYy9ibWFsbG9j L0dpZ2FjYWdlLmgKaW5kZXggMDI3ZTI4MGExODQyYWY2MzM4ODNiNDRlZDYxNDgxNTFiOTI5MTQx NS4uNzgzOWY4NmQ2MWIwN2IyNmFhNzYwYjcwOTIwOTUxNjliMTZhMjUwNSAxMDA2NDQKLS0tIGEv U291cmNlL2JtYWxsb2MvYm1hbGxvYy9HaWdhY2FnZS5oCisrKyBiL1NvdXJjZS9ibWFsbG9jL2Jt YWxsb2MvR2lnYWNhZ2UuaApAQCAtNDMsNiArNDMsMTMgQEAKICNkZWZpbmUgR0lHQUNBR0VfQUxM T0NBVElPTl9DQU5fRkFJTCAwCiAjZW5kaWYKIAorLy8gSW4gTGludXgsIGlmIGB2bS5vdmVyY29t bWl0X21lbW9yeSA9IDJgIGlzIHNwZWNpZmllZCwgbW1hcCB3aXRoIGxhcmdlIHNpemUgY2FuIGZh aWwgaWYgaXQgZXhjZWVkcyB0aGUgc2l6ZSBvZiBSQU0uCisvLyBTbyB3ZSBzcGVjaWZ5IEdJR0FD QUdFX0FMTE9DQVRJT05fQ0FOX0ZBSUwgPSAxLgorI2lmIEJPUyhMSU5VWCkKKyN1bmRlZiBHSUdB Q0FHRV9BTExPQ0FUSU9OX0NBTl9GQUlMCisjZGVmaW5lIEdJR0FDQUdFX0FMTE9DQVRJT05fQ0FO X0ZBSUwgMQorI2VuZGlmCisKIHN0YXRpY19hc3NlcnQoYm1hbGxvYzo6aXNQb3dlck9mVHdvKFBS SU1JVElWRV9HSUdBQ0FHRV9TSVpFKSwgIiIpOwogc3RhdGljX2Fzc2VydChibWFsbG9jOjppc1Bv d2VyT2ZUd28oSlNWQUxVRV9HSUdBQ0FHRV9TSVpFKSwgIiIpOwogCg==