Fix GNU3 demangler substitution table and decltype handling

plafosse · claude · plafosse · commit cfa2f0008146 · 2026-03-26T08:51:58.000-04:00
- Increase MAX_DEMANGLE_LENGTH to 262144 (256KB) to handle deeply
  templated symbols from real-world binaries

- Fix substitution table off-by-one for template functions whose
  template args are all types (no non-type L/X args): DemangleName's
  'N' case now pushes the complete specialized qualified name to the
  substitution table, matching the compiler's encoding. Introduced
  DemangleTemplateArgs(hadNonTypeArg) and DemangleNestedName(allTypeTemplateArgs)
  out-parameters to detect when all template args are types and the
  push is appropriate.

- Fix DemangleName's 'Z' (local-name) context: added m_inLocalName flag
  set around the inner DemangleSymbol call in DemangleLocalName, so the
  push only fires for the outer function name, not inner scope functions.

- Fix DemangleUnresolvedType decltype handling: the DT/Dt branch was
  peeking for the prefix but not consuming it before calling
  DemangleExpression, causing the expression parser to see 'D','T' with
  no matching case. Now correctly consumes the two-char prefix and the
  mandatory trailing 'E', enabling sr(decltype(...))::member patterns.

- Add forward template ref support for cv conversion operator types:
  ResolveForwardTemplateRefs patches placeholder types in the result
  once the enclosing template-args are known.

Co-Authored-By: Claude Sonnet 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/demangler/gnu3/demangle_gnu3.cpp b/demangler/gnu3/demangle_gnu3.cpp
@@ -30,7 +30,7 @@ using namespace std;
 #endif
 
 
-#define MAX_DEMANGLE_LENGTH    4096
+#define MAX_DEMANGLE_LENGTH    262144
 #define hash(x,y) (64 * x + y)
 
 #undef GNUDEMANGLE_DEBUG
@@ -308,7 +308,8 @@ DemangleGNU3::DemangleGNU3(Architecture* arch, const string& mangledName) :
 	m_isParameter(false),
 	m_shouldDeleteReader(true),
 	m_topLevel(true),
-	m_isOperatorOverload(false)
+	m_isOperatorOverload(false),
+	m_permitForwardTemplateRefs(false)
 {
 	MyLogDebug("%s : %s\n", __FUNCTION__, m_reader.GetRaw().c_str());
 }
@@ -330,6 +331,9 @@ void DemangleGNU3::Reset(Architecture* arch, const string& mangledName)
 	m_shouldDeleteReader = true;
 	m_topLevel = true;
 	m_isOperatorOverload = false;
+	m_permitForwardTemplateRefs = false;
+	m_pendingForwardRefs.clear();
+	m_inLocalName = false;
 }
 
 
@@ -369,10 +373,7 @@ void DemangleGNU3::PushTemplateType(const DemangledTypeNode& type)
 const DemangledTypeNode& DemangleGNU3::GetTemplateType(size_t ref)
 {
 	if (ref >= m_templateSubstitute.size())
-	{
-		// PrintTables();
 		throw DemangleException();
-	}
 	return m_templateSubstitute[ref];
 }
 
@@ -386,10 +387,7 @@ void DemangleGNU3::PushType(const DemangledTypeNode& type)
 const DemangledTypeNode& DemangleGNU3::GetType(size_t ref)
 {
 	if (ref >= m_substitute.size())
-	{
-		// PrintTables();
 		throw DemangleException();
-	}
 	return m_substitute[ref];
 }
 
@@ -482,7 +480,43 @@ DemangledTypeNode DemangleGNU3::DemangleFunction(bool cnst, bool vltl)
 }
 
 
-const DemangledTypeNode& DemangleGNU3::DemangleTemplateSubstitution()
+string DemangleGNU3::ForwardRefPlaceholder(size_t index)
+{
+	return "\x01FWDREF:" + to_string(index) + "\x01";
+}
+
+
+void DemangleGNU3::ResolveForwardTemplateRefs(DemangledTypeNode& type, const vector<string>& args)
+{
+	if (m_pendingForwardRefs.empty())
+		return;
+	auto& segs = type.GetMutableTypeName();
+	bool resolved = false;
+	for (const auto& fr : m_pendingForwardRefs)
+	{
+		string placeholder = ForwardRefPlaceholder(fr.index);
+		string replacement = (fr.index < args.size()) ? args[fr.index] : "auto";
+		for (auto& seg : segs)
+		{
+			size_t pos;
+			while ((pos = seg.find(placeholder)) != string::npos)
+			{
+				seg.replace(pos, placeholder.size(), replacement);
+				resolved = true;
+			}
+		}
+	}
+	// Only clear the pending list when we actually resolved something.  Inner
+	// nested-name 'I' handlers (e.g. template args of types nested inside the
+	// cv-operator result type) may call here with a type that does not contain
+	// the placeholder; we must not discard the pending entry in that case so
+	// that the correct outer 'I' handler can still resolve it.
+	if (resolved)
+		m_pendingForwardRefs.clear();
+}
+
+
+DemangledTypeNode DemangleGNU3::DemangleTemplateSubstitution()
 {
 	indent();
 	MyLogDebug("%s : %s\n", __FUNCTION__, m_reader.GetRaw().c_str());
@@ -512,7 +546,20 @@ const DemangledTypeNode& DemangleGNU3::DemangleTemplateSubstitution()
 		throw DemangleException();
 	}
 	dedent();
-	return GetTemplateType(number);
+
+	if (number < m_templateSubstitute.size())
+		return m_templateSubstitute[number];
+
+	// If forward template references are permitted (e.g. inside a cv conversion
+	// operator type), return a placeholder that will be resolved once the outer
+	// template args are known.
+	if (m_permitForwardTemplateRefs)
+	{
+		m_pendingForwardRefs.push_back({number});
+		return CreateUnknownType(ForwardRefPlaceholder(number));
+	}
+
+	throw DemangleException();
 }
 
 
@@ -615,8 +662,11 @@ DemangledTypeNode DemangleGNU3::DemangleType()
 
 		//Template Substitution
 		type = DemangleTemplateSubstitution();
-		substitute = true;
-		if (m_reader.Peek() == 'I')
+		// In forward-ref mode (cv conversion operator type parsing), do not consume
+		// trailing I<args>E — it belongs to the enclosing nested-name and will be
+		// processed by DemangleNestedName's 'I' case, which resolves forward refs.
+		substitute = !m_permitForwardTemplateRefs;
+		if (!m_permitForwardTemplateRefs && m_reader.Peek() == 'I')
 		{
 			m_reader.Consume();
 			if (substitute)
@@ -884,7 +934,11 @@ DemangledTypeNode DemangleGNU3::DemangleSubstitution()
 		}
 
 		dedent();
-		return GetType(number);
+		const DemangledTypeNode& resolved = GetType(number);
+		const auto& segs = resolved.GetTypeName();
+		if (!segs.empty())
+			m_lastName = segs.back();
+		return resolved;
 	}
 	m_lastName = name.back();
 	dedent();
@@ -1273,8 +1327,22 @@ DemangledTypeNode DemangleGNU3::DemangleUnqualifiedName()
 		break;
 	}
 	case hash('c','v'): //type (expression)
-		outType = CreateUnknownType("operator " + DemangleType().GetString());
+	{
+		// The conversion operator type may reference template params (T_, T0_, ...)
+		// that aren't yet in m_templateSubstitute (they're defined by a following
+		// I<args>E in the enclosing nested name).  Set m_permitForwardTemplateRefs so
+		// that DemangleTemplateSubstitution() returns a placeholder instead of
+		// throwing, and don't consume trailing I<args>E in the T case of DemangleType.
+		// The outer DemangleNestedName case 'I' will parse those args and call
+		// ResolveForwardTemplateRefs() to patch the placeholders.
+		bool savedPermit = m_permitForwardTemplateRefs;
+		m_pendingForwardRefs.clear();
+		m_permitForwardTemplateRefs = true;
+		DemangledTypeNode cvType = DemangleType();
+		m_permitForwardTemplateRefs = savedPermit;
+		outType = CreateUnknownType("operator " + cvType.GetString());
 		break;
+	}
 	default:
 		m_reader.UnRead(2);
 		if (isdigit(m_reader.Peek()) || m_reader.Read() == 'L')
@@ -1368,10 +1436,21 @@ DemangledTypeNode DemangleGNU3::DemangleUnresolvedType()
 			type.SetHasTemplateArguments(true);
 			PushType(type);
 		}
+		else
+		{
+			// Template param used as scope qualifier (e.g. sr T_ name) is a substitution
+			// candidate: the compiler adds it to the main sub table so subsequent
+			// occurrences can use Sn_ instead of T_.
+			PushType(type);
+		}
 	}
 	else if (m_reader.Length() > 2 && (m_reader.PeekString(2) == "Dt" || m_reader.PeekString(2) == "DT"))
 	{
+		m_reader.Consume(); // 'D'
+		m_reader.Consume(); // 't' or 'T'
 		const string name = "decltype(" + DemangleExpression() + ")";
+		if (m_reader.Read() != 'E')
+			throw DemangleException();
 		type = CreateUnknownType(name);
 	}
 	else if (m_reader.Peek() == 'S')
@@ -1731,7 +1810,7 @@ string DemangleGNU3::DemangleExpression()
 }
 
 
-void DemangleGNU3::DemangleTemplateArgs(vector<string>& args)
+void DemangleGNU3::DemangleTemplateArgs(vector<string>& args, bool* hadNonTypeArg)
 {
 	indent();
 	MyLogDebug("%s:: '%s'\n", __FUNCTION__, m_reader.GetRaw().c_str());
@@ -1749,16 +1828,23 @@ void DemangleGNU3::DemangleTemplateArgs(vector<string>& args)
 			args.push_back(expr);
 			tmp = CreateUnknownType(expr);
 			tmpValid = true;
+			if (hadNonTypeArg) *hadNonTypeArg = true;
 			break;
 		case 'X':
 			args.push_back(DemangleExpression());
 			if (m_reader.Read() != 'E')
 				throw DemangleException();
+			if (hadNonTypeArg) *hadNonTypeArg = true;
 			break;
 		case 'I': // GCC sometimes uses I...E for argument packs instead of J...E
 		case 'J':
+		{
+			size_t prevTemplateSize = m_templateSubstitute.size();
 			DemangleTemplateArgs(args);
+			if (m_topLevel && m_templateSubstitute.size() == prevTemplateSize)
+				PushTemplateType(CreateUnknownType("auto"));
 			break;
+		}
 		default:
 			m_reader.UnRead();
 			topLevel = m_topLevel;
@@ -1781,7 +1867,7 @@ void DemangleGNU3::DemangleTemplateArgs(vector<string>& args)
 }
 
 
-DemangledTypeNode DemangleGNU3::DemangleNestedName()
+DemangledTypeNode DemangleGNU3::DemangleNestedName(bool* allTypeTemplateArgs)
 {
 	/*
 	This can be either a qualified name like: "foo::bar::bas"
@@ -1858,7 +1944,16 @@ DemangledTypeNode DemangleGNU3::DemangleNestedName()
 			if (!base)
 				throw DemangleException();
 			vector<string> args;
-			DemangleTemplateArgs(args);
+			bool hadNonType = false;
+			DemangleTemplateArgs(args, allTypeTemplateArgs ? &hadNonType : nullptr);
+			if (allTypeTemplateArgs)
+				*allTypeTemplateArgs = !hadNonType;
+			// Resolve any forward template refs created while parsing a cv
+			// conversion operator type (e.g. cv T_ where T_ wasn't yet known).
+			// Only do this in the outer context (not while still inside the cv
+			// type parsing itself where m_permitForwardTemplateRefs is true).
+			if (!m_permitForwardTemplateRefs)
+				ResolveForwardTemplateRefs(type, args);
 			ExtendTypeName(type, GetTemplateString(args));
 			type.SetHasTemplateArguments(true);
 			isTemplate = true;
@@ -1948,7 +2043,10 @@ DemangledTypeNode DemangleGNU3::DemangleLocalName()
 	m_templateSubstitute.clear();
 	bool oldTopLevel = m_topLevel;
 	m_topLevel = true;
+	bool savedInLocalName = m_inLocalName;
+	m_inLocalName = true;
 	type = DemangleSymbol(varName);
+	m_inLocalName = savedInLocalName;
 	m_topLevel = oldTopLevel;
 	m_templateSubstitute = std::move(savedTemplateSubstitute);
 
@@ -1959,6 +2057,15 @@ DemangledTypeNode DemangleGNU3::DemangleLocalName()
 
 	if (m_reader.Peek() != 's')
 	{
+		// Handle default argument context: d [<number>] _ <name>
+		if (m_reader.Peek() == 'd')
+		{
+			m_reader.Consume();
+			if (isdigit(m_reader.Peek()))
+				DemangleNumber();
+			if (m_reader.Peek() == '_')
+				m_reader.Consume();
+		}
 		//<entity name>
 		DemangledTypeNode tmpType = DemangleName();
 		type = DemangledTypeNode::NamedType(UnknownNamedTypeClass, varName);
@@ -2049,8 +2156,13 @@ DemangledTypeNode DemangleGNU3::DemangleName()
 		}
 		break;
 	case 'N': //<nested-name>
-		type = DemangleNestedName();
+	{
+		bool allTypeArgs = false;
+		type = DemangleNestedName(&allTypeArgs);
+		if (!m_inLocalName && allTypeArgs)
+			PushType(type);
 		break;
+	}
 	case 'Z': //<local-name>
 		type = DemangleLocalName();
 		break;
diff --git a/demangler/gnu3/demangle_gnu3.h b/demangler/gnu3/demangle_gnu3.h
@@ -109,6 +109,17 @@ class DemangleGNU3
 	bool m_shouldDeleteReader;
 	bool m_topLevel;
 	bool m_isOperatorOverload;
+	// Forward template reference support (for cv conversion operator types).
+	// When m_permitForwardTemplateRefs is true, DemangleTemplateSubstitution()
+	// returns a placeholder instead of throwing for out-of-bounds template params.
+	// m_pendingForwardRefs records which param indices have placeholders so that
+	// ResolveForwardTemplateRefs() can patch them once template args are known.
+	bool m_permitForwardTemplateRefs;
+	bool m_inLocalName;
+	struct ForwardRef { size_t index; };
+	_STD_VECTOR<ForwardRef> m_pendingForwardRefs;
+	void ResolveForwardTemplateRefs(DemangledTypeNode& type, const _STD_VECTOR<_STD_STRING>& args);
+	static _STD_STRING ForwardRefPlaceholder(size_t index);
 	enum SymbolType { Function, FunctionWithReturn, Data, VTable, Rtti, Name};
 	BN::QualifiedName DemangleBaseUnresolvedName();
 	DemangledTypeNode DemangleUnresolvedType();
@@ -129,12 +140,12 @@ class DemangleGNU3
 
 	void DemangleCVQualifiers(bool& cnst, bool& vltl, bool& rstrct);
 	DemangledTypeNode DemangleSubstitution();
-	const DemangledTypeNode& DemangleTemplateSubstitution();
-	void DemangleTemplateArgs(_STD_VECTOR<_STD_STRING>& args);
+	DemangledTypeNode DemangleTemplateSubstitution();
+	void DemangleTemplateArgs(_STD_VECTOR<_STD_STRING>& args, bool* hadNonTypeArg = nullptr);
 	DemangledTypeNode DemangleFunction(bool cnst, bool vltl);
 	DemangledTypeNode DemangleType();
 	int64_t DemangleNumber();
-	DemangledTypeNode DemangleNestedName();
+	DemangledTypeNode DemangleNestedName(bool* allTypeTemplateArgs = nullptr);
 	void PushTemplateType(const DemangledTypeNode& type);
 	const DemangledTypeNode& GetTemplateType(size_t ref);
 	void PushType(const DemangledTypeNode& type);
