wasm-ctor-eval: Do not try to flatten segments with huge offsets

src/ir/memory-utils.cpp

@@ -20,6 +20,10 @@
 
 namespace wasm::MemoryUtils {
 
+// When flattening, do not generate a massive segment that will likely just
+// OOM.
+static uint64_t MaxFlatMemorySize = 4ULL * 1024 * 1024 * 1024;
+
 bool isSubType(const Memory& a, const Memory& b) {
   return a.shared == b.shared && a.addressType == b.addressType &&
          a.initial >= b.initial && a.max <= b.max &&
@@ -104,13 +108,16 @@ bool flatten(Module& wasm) {
   }
   for (auto& segment : dataSegments) {
     auto* offset = segment->offset->dynCast<Const>();
-    Index start = offset->value.getInteger();
-    Index size = segment->data.size();
-    Index end;
+    uint64_t start = offset->value.getUnsigned();
+    uint64_t size = segment->data.size();
+    uint64_t end;
     if (std::ckd_add(&end, start, size)) {
       return false;
     }
     if (end > data.size()) {
+      if (end > MaxFlatMemorySize) {
+        return false;
+      }
       data.resize(end);
     }
     std::copy(segment->data.begin(), segment->data.end(), data.begin() + start);

test/lit/ctor-eval/memory64-massive.wast

@@ -0,0 +1,24 @@
+;; NOTE: Assertions have been generated by update_lit_checks.py --all-items and should not be edited.
+;; RUN: wasm-ctor-eval %s --ctors=func --kept-exports=func --quiet -all -S -o - | filecheck %s
+
+;; The empty segment here has a huge offset. We do not flatten it into a
+;; segment full of zeroes that starts at 0 (the flat form), since that would
+;; OOM, so we fail to eval the ctor here.
+
+(module
+ ;; CHECK:      (type $0 (func))
+
+ ;; CHECK:      (memory $mem i64 16 16 shared)
+ (memory $mem i64 16 16 shared)
+
+ ;; CHECK:      (data $data (i64.const 4294968321) "")
+ (data $data (i64.const 4294968321) "")
+
+ ;; CHECK:      (export "func" (func $func))
+
+ ;; CHECK:      (func $func (type $0)
+ ;; CHECK-NEXT: )
+ (func $func (export "func")
+ )
+)
+