feat: parallel dag creation with test fixes and label dependency removal fixes

Author: f7f376a1fcd0d0e11a10ed1b6577c9

dag/dag.go

@@ -1,14 +1,13 @@
 package dag
 
 import (
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
 )
 
 func TestFull(t *testing.T) {
-	tmpDir, err := ioutil.TempDir("", "test")
+	tmpDir, err := os.MkdirTemp("", "test")
 	if err != nil {
 		t.Fatalf("Could not create temp directory: %s", err)
 	}

dag/dag_test.go

@@ -2,7 +2,6 @@ package dag
 
 import (
 	"bytes"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -11,15 +10,15 @@ import (
 
 func TestOutOfRangeLeafRequests(t *testing.T) {
 	// Create a simple DAG with known number of leaves
-	tmpDir, err := ioutil.TempDir("", "test")
+	tmpDir, err := os.MkdirTemp("", "test")
 	if err != nil {
 		t.Fatalf("Could not create temp directory: %s", err)
 	}
 	defer os.RemoveAll(tmpDir)
 
 	// Create 5 test files
 	for i := 0; i < 5; i++ {
-		err := ioutil.WriteFile(
+		err := os.WriteFile(
 			filepath.Join(tmpDir, string(rune('a'+i))),
 			[]byte("test content"),
 			0644,
@@ -61,7 +60,7 @@ func TestOutOfRangeLeafRequests(t *testing.T) {
 }
 
 func TestSingleFileScenarios(t *testing.T) {
-	tmpDir, err := ioutil.TempDir("", "test")
+	tmpDir, err := os.MkdirTemp("", "test")
 	if err != nil {
 		t.Fatalf("Could not create temp directory: %s", err)
 	}
@@ -113,7 +112,7 @@ func TestSingleFileScenarios(t *testing.T) {
 			}
 
 			// Create the test file
-			err := ioutil.WriteFile(filePath, content, 0644)
+			err := os.WriteFile(filePath, content, 0644)
 			if err != nil {
 				t.Fatalf("Failed to create test file: %v", err)
 			}

dag/edge_test.go

@@ -30,7 +30,7 @@ func TestStripLabel(t *testing.T) {
 		{
 			name:     "Hash with multiple colons",
 			input:    "1:2:3",
-			expected: "1:2:3",
+			expected: "2:3",
 		},
 	}
 

dag/label_test.go

@@ -66,19 +66,6 @@ func (b *DagBuilder) GetLatestLabel() string {
 	return result
 }
 
-func (b *DagBuilder) GetNextAvailableLabel() string {
-	latestLabel := b.GetLatestLabel()
-
-	number, err := strconv.ParseInt(latestLabel, 10, 64)
-	if err != nil {
-		fmt.Println("Failed to parse label")
-	}
-
-	nextLabel := strconv.FormatInt(number+1, 10)
-
-	return nextLabel
-}
-
 // CalculateTotalContentSize calculates the total size of actual content (not including metadata)
 func (b *DagBuilder) CalculateTotalContentSize() int64 {
 	var totalSize int64
@@ -94,8 +81,50 @@ func (b *DagBuilder) CalculateTotalContentSize() int64 {
 func (b *DagBuilder) CalculateTotalDagSize() (int64, error) {
 	var totalSize int64
 	for _, leaf := range b.Leafs {
-		serializable := leaf.ToSerializable()
-		serialized, err := cbor.Marshal(serializable)
+		bareHash := GetHash(leaf.Hash)
+
+		var linkHashes []string
+		if len(leaf.Links) > 0 {
+			linkHashes = make([]string, 0, len(leaf.Links))
+			for _, linkHash := range leaf.Links {
+				linkHashes = append(linkHashes, GetHash(linkHash))
+			}
+			sort.Strings(linkHashes)
+		}
+
+		data := struct {
+			Hash              string
+			ItemName          string
+			Type              LeafType
+			ContentHash       []byte
+			Content           []byte
+			ClassicMerkleRoot []byte
+			CurrentLinkCount  int
+			LatestLabel       string
+			LeafCount         int
+			ContentSize       int64
+			DagSize           int64
+			Links             []string
+			AdditionalData    map[string]string
+			StoredProofs      map[string]*ClassicTreeBranch
+		}{
+			Hash:              bareHash,
+			ItemName:          leaf.ItemName,
+			Type:              leaf.Type,
+			ContentHash:       leaf.ContentHash,
+			Content:           leaf.Content,
+			ClassicMerkleRoot: leaf.ClassicMerkleRoot,
+			CurrentLinkCount:  leaf.CurrentLinkCount,
+			LatestLabel:       leaf.LatestLabel,
+			LeafCount:         leaf.LeafCount,
+			ContentSize:       leaf.ContentSize,
+			DagSize:           leaf.DagSize,
+			Links:             linkHashes,
+			AdditionalData:    sortMapByKeys(leaf.AdditionalData),
+			StoredProofs:      leaf.Proofs,
+		}
+
+		serialized, err := cbor.Marshal(data)
 		if err != nil {
 			return 0, fmt.Errorf("failed to serialize leaf %s: %w", leaf.Hash, err)
 		}
@@ -253,7 +282,7 @@ func (b *DagLeafBuilder) BuildRootLeaf(dag *DagBuilder, additionalData map[strin
 		MerkleRoot:       merkleRoot,
 		CurrentLinkCount: len(b.Links),
 		LatestLabel:      latestLabel,
-		LeafCount:        len(dag.Leafs),
+		LeafCount:        len(dag.Leafs) + 1,
 		ContentSize:      contentSize,
 		DagSize:          0,
 		ContentHash:      nil,
@@ -525,7 +554,7 @@ func (leaf *DagLeaf) VerifyLeaf() error {
 func (leaf *DagLeaf) VerifyRootLeaf(dag *Dag) error {
 	additionalData := sortMapByKeys(leaf.AdditionalData)
 
-	if leaf.ClassicMerkleRoot == nil || len(leaf.ClassicMerkleRoot) <= 0 {
+	if len(leaf.ClassicMerkleRoot) <= 0 {
 		leaf.ClassicMerkleRoot = []byte{}
 	}
 
@@ -536,6 +565,16 @@ func (leaf *DagLeaf) VerifyRootLeaf(dag *Dag) error {
 		len(dag.Leafs) == leaf.LeafCount &&
 		(leaf.ContentSize != 0 || leaf.DagSize != 0 || leaf.LeafCount == 1)
 
+	// If any leaf has pruned links, this is a partial DAG
+	if isFullDag && dag != nil {
+		for _, l := range dag.Leafs {
+			if len(l.Links) < l.CurrentLinkCount {
+				isFullDag = false
+				break
+			}
+		}
+	}
+
 	if isFullDag {
 		for _, dagLeaf := range dag.Leafs {
 			if dagLeaf.Content != nil {
@@ -550,8 +589,50 @@ func (leaf *DagLeaf) VerifyRootLeaf(dag *Dag) error {
 				continue
 			}
 
-			serializable := dagLeaf.ToSerializable()
-			serialized, err := cbor.Marshal(serializable)
+			bareHash := GetHash(dagLeaf.Hash)
+
+			var linkHashes []string
+			if len(dagLeaf.Links) > 0 {
+				linkHashes = make([]string, 0, len(dagLeaf.Links))
+				for _, linkHash := range dagLeaf.Links {
+					linkHashes = append(linkHashes, GetHash(linkHash))
+				}
+				sort.Strings(linkHashes)
+			}
+
+			data := struct {
+				Hash              string
+				ItemName          string
+				Type              LeafType
+				ContentHash       []byte
+				Content           []byte
+				ClassicMerkleRoot []byte
+				CurrentLinkCount  int
+				LatestLabel       string
+				LeafCount         int
+				ContentSize       int64
+				DagSize           int64
+				Links             []string
+				AdditionalData    map[string]string
+				StoredProofs      map[string]*ClassicTreeBranch
+			}{
+				Hash:              bareHash,
+				ItemName:          dagLeaf.ItemName,
+				Type:              dagLeaf.Type,
+				ContentHash:       dagLeaf.ContentHash,
+				Content:           dagLeaf.Content,
+				ClassicMerkleRoot: dagLeaf.ClassicMerkleRoot,
+				CurrentLinkCount:  dagLeaf.CurrentLinkCount,
+				LatestLabel:       dagLeaf.LatestLabel,
+				LeafCount:         dagLeaf.LeafCount,
+				ContentSize:       dagLeaf.ContentSize,
+				DagSize:           dagLeaf.DagSize,
+				Links:             linkHashes,
+				AdditionalData:    sortMapByKeys(dagLeaf.AdditionalData),
+				StoredProofs:      dagLeaf.Proofs,
+			}
+
+			serialized, err := cbor.Marshal(data)
 			if err != nil {
 				return fmt.Errorf("failed to serialize leaf %s for size verification: %w", dagLeaf.Hash, err)
 			}
@@ -830,10 +911,10 @@ func GetLabel(hash string) string {
 // StripLabel removes the "label:" prefix from a hash string
 func StripLabel(hash string) string {
 	parts := strings.Split(hash, ":")
-	if len(parts) != 2 {
+	if len(parts) < 2 {
 		return hash
 	}
-	return parts[1]
+	return strings.Join(parts[1:], ":")
 }
 
 // ReplaceLabelInLink replaces the label in a child's link hash

dag/leaves.go

@@ -1,7 +1,6 @@
 package dag
 
 import (
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"testing"
@@ -10,7 +9,7 @@ import (
 // TestMerkleRootVerificationDetectsTampering verifies that our enhanced verification
 // detects when children are tampered with even if the parent leaf hash is valid
 func TestMerkleRootVerificationDetectsTampering(t *testing.T) {
-	tmpDir, err := ioutil.TempDir("", "merkle_verification_test")
+	tmpDir, err := os.MkdirTemp("", "merkle_verification_test")
 	if err != nil {
 		t.Fatalf("Could not create temp directory: %s", err)
 	}
@@ -202,7 +201,7 @@ func TestMerkleRootVerificationDetectsTampering(t *testing.T) {
 
 // TestPartialDagMerkleVerification ensures that partial DAGs still work correctly
 func TestPartialDagMerkleVerification(t *testing.T) {
-	tmpDir, err := ioutil.TempDir("", "partial_dag_merkle_test")
+	tmpDir, err := os.MkdirTemp("", "partial_dag_merkle_test")
 	if err != nil {
 		t.Fatalf("Could not create temp directory: %s", err)
 	}
@@ -242,12 +241,21 @@ func TestPartialDagMerkleVerification(t *testing.T) {
 	// Use a range that's guaranteed to be within bounds
 	// GetPartial uses 1-indexed leaf positions, and end must be <= rootLeaf.LeafCount
 	rootLeaf := fullDag.Leafs[fullDag.Root]
-	maxRange := rootLeaf.LeafCount
+
+	// Make sure we actually create a PARTIAL dag, not a full one
+	// We need at least 3 leaves total to create a meaningful partial (1 root + 2+ children)
+	if rootLeaf.LeafCount < 3 {
+		t.Skip("Need at least 3 leaves (1 root + 2 children) to create a partial DAG")
+	}
+
+	// Request only SOME of the leaves, not all
+	// Cap at min(5, LeafCount-1) to ensure we're always leaving at least 1 leaf out
+	maxRange := rootLeaf.LeafCount - 1 // Always exclude at least one leaf
 	if maxRange > 5 {
 		maxRange = 5
 	}
 	if maxRange < 2 {
-		maxRange = 2 // Need at least 2 leaves for a valid range
+		maxRange = 2
 	}
 
 	partialDag, err := fullDag.GetPartial(1, maxRange)

dag/merkle_verification_test.go

@@ -0,0 +1,67 @@
+package dag
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// TestSimpleParallelDeterminism tests with a very simple directory structure
+func TestSimpleParallelDeterminism(t *testing.T) {
+	// Create temporary test directory structure
+	tmpDir, err := os.MkdirTemp("", "simple-parallel-test-*")
+	if err != nil {
+		t.Fatalf("Failed to create temp dir: %v", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	// Create simple structure:
+	// tmpDir/
+	//   ├── a.txt
+	//   └── b.txt
+
+	if err := os.WriteFile(filepath.Join(tmpDir, "a.txt"), []byte("a"), 0644); err != nil {
+		t.Fatalf("Failed to write file: %v", err)
+	}
+	if err := os.WriteFile(filepath.Join(tmpDir, "b.txt"), []byte("b"), 0644); err != nil {
+		t.Fatalf("Failed to write file: %v", err)
+	}
+
+	// Build DAG sequentially
+	sequentialConfig := DefaultConfig()
+	sequentialDAG, err := CreateDagWithConfig(tmpDir, sequentialConfig)
+	if err != nil {
+		t.Fatalf("Sequential DAG creation failed: %v", err)
+	}
+
+	t.Logf("Sequential DAG root: %s", sequentialDAG.Root)
+	t.Logf("Sequential DAG leaves:")
+	for label, leaf := range sequentialDAG.Leafs {
+		t.Logf("  %s: %s (type=%s, name=%s, links=%d)", label, leaf.Hash, leaf.Type, leaf.ItemName, len(leaf.Links))
+		for linkLabel, linkHash := range leaf.Links {
+			t.Logf("    -> %s: %s", linkLabel, linkHash)
+		}
+	}
+
+	// Build DAG in parallel
+	parallelConfig := ParallelConfigWithWorkers(2)
+	parallelDAG, err := CreateDagWithConfig(tmpDir, parallelConfig)
+	if err != nil {
+		t.Fatalf("Parallel DAG creation failed: %v", err)
+	}
+
+	t.Logf("\nParallel DAG root: %s", parallelDAG.Root)
+	t.Logf("Parallel DAG leaves:")
+	for label, leaf := range parallelDAG.Leafs {
+		t.Logf("  %s: %s (type=%s, name=%s, links=%d)", label, leaf.Hash, leaf.Type, leaf.ItemName, len(leaf.Links))
+		for linkLabel, linkHash := range leaf.Links {
+			t.Logf("    -> %s: %s", linkLabel, linkHash)
+		}
+	}
+
+	// Compare
+	if sequentialDAG.Root != parallelDAG.Root {
+		t.Errorf("Root hashes don't match!\nSequential: %s\nParallel:   %s",
+			sequentialDAG.Root, parallelDAG.Root)
+	}
+}