Skip to content

Commit bcfe5b2

Browse files
f7f376a1fcd0d0e11a10ed1b6577c9f7f376a1fcd0d0e11a10ed1b6577c9
committed
feat: added dag size and content size to root leaf
1 parent b8a0b08 commit bcfe5b2

4 files changed

Lines changed: 216 additions & 23 deletions

File tree

‎dag/dag.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,7 @@ func (b *DagBuilder) BuildDag(root string) *Dag {
330330
func (d *Dag) verifyFullDag() error {
331331
return d.IterateDag(func(leaf *DagLeaf, parent *DagLeaf) error {
332332
if leaf.Hash == d.Root {
333-
err := leaf.VerifyRootLeaf()
333+
err := leaf.VerifyRootLeaf(d)
334334
if err != nil {
335335
return err
336336
}
@@ -366,7 +366,7 @@ func (d *Dag) verifyFullDag() error {
366366
func (d *Dag) verifyWithProofs() error {
367367
// First verify the root leaf
368368
rootLeaf := d.Leafs[d.Root]
369-
if err := rootLeaf.VerifyRootLeaf(); err != nil {
369+
if err := rootLeaf.VerifyRootLeaf(d); err != nil {
370370
return fmt.Errorf("root leaf failed to verify: %w", err)
371371
}
372372

@@ -1000,7 +1000,7 @@ func (d *Dag) GetLeafSequence() []*TransmissionPacket {
10001000
// VerifyTransmissionPacket verifies a transmission packet independently
10011001
func (d *Dag) VerifyTransmissionPacket(packet *TransmissionPacket) error {
10021002
if packet.ParentHash == "" {
1003-
if err := packet.Leaf.VerifyRootLeaf(); err != nil {
1003+
if err := packet.Leaf.VerifyRootLeaf(nil); err != nil {
10041004
return fmt.Errorf("transmission packet root leaf verification failed: %w", err)
10051005
}
10061006
} else {
@@ -1366,7 +1366,7 @@ func (d *Dag) VerifyBatchedTransmissionPacket(packet *BatchedTransmissionPacket)
13661366
}
13671367

13681368
if parentHash == "" {
1369-
if err := childLeaf.VerifyRootLeaf(); err != nil {
1369+
if err := childLeaf.VerifyRootLeaf(nil); err != nil {
13701370
return fmt.Errorf("batched transmission packet root leaf %s verification failed: %w", childHash, err)
13711371
}
13721372
} else {

‎dag/leaves.go‎

Lines changed: 153 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,31 @@ func (b *DagBuilder) GetNextAvailableLabel() string {
7979
return nextLabel
8080
}
8181

82+
// CalculateTotalContentSize calculates the total size of actual content (not including metadata)
83+
func (b *DagBuilder) CalculateTotalContentSize() int64 {
84+
var totalSize int64
85+
for _, leaf := range b.Leafs {
86+
if leaf.Content != nil {
87+
totalSize += int64(len(leaf.Content))
88+
}
89+
}
90+
return totalSize
91+
}
92+
93+
// CalculateTotalDagSize calculates the total size of all serialized leaves in the DAG
94+
func (b *DagBuilder) CalculateTotalDagSize() (int64, error) {
95+
var totalSize int64
96+
for _, leaf := range b.Leafs {
97+
serializable := leaf.ToSerializable()
98+
serialized, err := cbor.Marshal(serializable)
99+
if err != nil {
100+
return 0, fmt.Errorf("failed to serialize leaf %s: %w", leaf.Hash, err)
101+
}
102+
totalSize += int64(len(serialized))
103+
}
104+
return totalSize, nil
105+
}
106+
82107
func (b *DagLeafBuilder) BuildLeaf(additionalData map[string]string) (*DagLeaf, error) {
83108
if b.LeafType == "" {
84109
err := fmt.Errorf("leaf must have a type defined")
@@ -201,15 +226,25 @@ func (b *DagLeafBuilder) BuildRootLeaf(dag *DagBuilder, additionalData map[strin
201226

202227
latestLabel := dag.GetLatestLabel()
203228

204-
additionalData = sortMapByKeys(additionalData)
229+
contentSize := dag.CalculateTotalContentSize()
230+
if b.Data != nil {
231+
contentSize += int64(len(b.Data))
232+
}
205233

206-
leafData := struct {
234+
childrenDagSize, err := dag.CalculateTotalDagSize()
235+
if err != nil {
236+
return nil, fmt.Errorf("failed to calculate dag size: %w", err)
237+
}
238+
239+
tempLeafData := struct {
207240
ItemName string
208241
Type LeafType
209242
MerkleRoot []byte
210243
CurrentLinkCount int
211244
LatestLabel string
212245
LeafCount int
246+
ContentSize int64
247+
DagSize int64
213248
ContentHash []byte
214249
AdditionalData []keyValue
215250
}{
@@ -219,13 +254,49 @@ func (b *DagLeafBuilder) BuildRootLeaf(dag *DagBuilder, additionalData map[strin
219254
CurrentLinkCount: len(b.Links),
220255
LatestLabel: latestLabel,
221256
LeafCount: len(dag.Leafs),
257+
ContentSize: contentSize,
258+
DagSize: 0,
222259
ContentHash: nil,
223260
AdditionalData: sortMapForVerification(additionalData),
224261
}
225262

226263
if b.Data != nil {
227264
hash := sha256.Sum256(b.Data)
228-
leafData.ContentHash = hash[:]
265+
tempLeafData.ContentHash = hash[:]
266+
}
267+
268+
tempSerialized, err := cbor.Marshal(tempLeafData)
269+
if err != nil {
270+
return nil, err
271+
}
272+
rootLeafSize := int64(len(tempSerialized))
273+
274+
finalDagSize := childrenDagSize + rootLeafSize
275+
276+
additionalData = sortMapByKeys(additionalData)
277+
278+
leafData := struct {
279+
ItemName string
280+
Type LeafType
281+
MerkleRoot []byte
282+
CurrentLinkCount int
283+
LatestLabel string
284+
LeafCount int
285+
ContentSize int64
286+
DagSize int64
287+
ContentHash []byte
288+
AdditionalData []keyValue
289+
}{
290+
ItemName: b.ItemName,
291+
Type: b.LeafType,
292+
MerkleRoot: merkleRoot,
293+
CurrentLinkCount: len(b.Links),
294+
LatestLabel: latestLabel,
295+
LeafCount: len(dag.Leafs),
296+
ContentSize: contentSize,
297+
DagSize: finalDagSize,
298+
ContentHash: tempLeafData.ContentHash, // Reuse from temp
299+
AdditionalData: sortMapForVerification(additionalData),
229300
}
230301

231302
serializedLeafData, err := cbor.Marshal(leafData)
@@ -254,6 +325,8 @@ func (b *DagLeafBuilder) BuildRootLeaf(dag *DagBuilder, additionalData map[strin
254325
CurrentLinkCount: len(b.Links),
255326
LatestLabel: latestLabel,
256327
LeafCount: len(dag.Leafs),
328+
ContentSize: contentSize,
329+
DagSize: finalDagSize,
257330
Content: b.Data,
258331
ContentHash: leafData.ContentHash,
259332
Links: sortedLinks,
@@ -449,20 +522,92 @@ func (leaf *DagLeaf) VerifyLeaf() error {
449522
return nil
450523
}
451524

452-
func (leaf *DagLeaf) VerifyRootLeaf() error {
525+
func (leaf *DagLeaf) VerifyRootLeaf(dag *Dag) error {
453526
additionalData := sortMapByKeys(leaf.AdditionalData)
454527

455528
if leaf.ClassicMerkleRoot == nil || len(leaf.ClassicMerkleRoot) <= 0 {
456529
leaf.ClassicMerkleRoot = []byte{}
457530
}
458531

532+
var calculatedContentSize int64
533+
var calculatedDagSize int64
534+
535+
isFullDag := dag != nil &&
536+
len(dag.Leafs) == leaf.LeafCount &&
537+
(leaf.ContentSize != 0 || leaf.DagSize != 0 || leaf.LeafCount == 1)
538+
539+
if isFullDag {
540+
for _, dagLeaf := range dag.Leafs {
541+
if dagLeaf.Content != nil {
542+
calculatedContentSize += int64(len(dagLeaf.Content))
543+
}
544+
}
545+
546+
rootHash := GetHash(leaf.Hash)
547+
var childrenDagSize int64
548+
for _, dagLeaf := range dag.Leafs {
549+
if GetHash(dagLeaf.Hash) == rootHash {
550+
continue
551+
}
552+
553+
serializable := dagLeaf.ToSerializable()
554+
serialized, err := cbor.Marshal(serializable)
555+
if err != nil {
556+
return fmt.Errorf("failed to serialize leaf %s for size verification: %w", dagLeaf.Hash, err)
557+
}
558+
childrenDagSize += int64(len(serialized))
559+
}
560+
561+
tempLeafData := struct {
562+
ItemName string
563+
Type LeafType
564+
MerkleRoot []byte
565+
CurrentLinkCount int
566+
LatestLabel string
567+
LeafCount int
568+
ContentSize int64
569+
DagSize int64
570+
ContentHash []byte
571+
AdditionalData []keyValue
572+
}{
573+
ItemName: leaf.ItemName,
574+
Type: leaf.Type,
575+
MerkleRoot: leaf.ClassicMerkleRoot,
576+
CurrentLinkCount: leaf.CurrentLinkCount,
577+
LatestLabel: leaf.LatestLabel,
578+
LeafCount: leaf.LeafCount,
579+
ContentSize: leaf.ContentSize,
580+
DagSize: 0,
581+
ContentHash: leaf.ContentHash,
582+
AdditionalData: sortMapForVerification(additionalData),
583+
}
584+
585+
tempSerialized, err := cbor.Marshal(tempLeafData)
586+
if err != nil {
587+
return fmt.Errorf("failed to serialize root leaf for size verification: %w", err)
588+
}
589+
rootLeafSize := int64(len(tempSerialized))
590+
591+
calculatedDagSize = childrenDagSize + rootLeafSize
592+
593+
if leaf.ContentSize != calculatedContentSize {
594+
return fmt.Errorf("content size mismatch: stored %d, calculated %d", leaf.ContentSize, calculatedContentSize)
595+
}
596+
597+
if leaf.DagSize != calculatedDagSize {
598+
return fmt.Errorf("dag size mismatch: stored %d, calculated %d", leaf.DagSize, calculatedDagSize)
599+
}
600+
}
601+
459602
leafData := struct {
460603
ItemName string
461604
Type LeafType
462605
MerkleRoot []byte
463606
CurrentLinkCount int
464607
LatestLabel string
465608
LeafCount int
609+
ContentSize int64
610+
DagSize int64
466611
ContentHash []byte
467612
AdditionalData []keyValue
468613
}{
@@ -472,6 +617,8 @@ func (leaf *DagLeaf) VerifyRootLeaf() error {
472617
CurrentLinkCount: leaf.CurrentLinkCount,
473618
LatestLabel: leaf.LatestLabel,
474619
LeafCount: leaf.LeafCount,
620+
ContentSize: leaf.ContentSize,
621+
DagSize: leaf.DagSize,
475622
ContentHash: leaf.ContentHash,
476623
AdditionalData: sortMapForVerification(additionalData),
477624
}
@@ -622,6 +769,8 @@ func (leaf *DagLeaf) Clone() *DagLeaf {
622769
CurrentLinkCount: leaf.CurrentLinkCount,
623770
LatestLabel: leaf.LatestLabel,
624771
LeafCount: leaf.LeafCount,
772+
ContentSize: leaf.ContentSize,
773+
DagSize: leaf.DagSize,
625774
ParentHash: leaf.ParentHash,
626775
Links: make(map[string]string),
627776
AdditionalData: make(map[string]string),

‎dag/serialize.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ type SerializableDagLeaf struct {
2424
CurrentLinkCount int
2525
LatestLabel string
2626
LeafCount int
27+
ContentSize int64
28+
DagSize int64
2729
Links map[string]string
2830
AdditionalData map[string]string
2931
StoredProofs map[string]*ClassicTreeBranch `json:"stored_proofs,omitempty" cbor:"stored_proofs,omitempty"`
@@ -96,6 +98,8 @@ func FromSerializable(s *SerializableDag) *Dag {
9698
if hash == s.Root {
9799
dag.Leafs[hash].LeafCount = sLeaf.LeafCount
98100
dag.Leafs[hash].LatestLabel = sLeaf.LatestLabel
101+
dag.Leafs[hash].ContentSize = sLeaf.ContentSize
102+
dag.Leafs[hash].DagSize = sLeaf.DagSize
99103
}
100104
}
101105

@@ -156,6 +160,8 @@ func (leaf *DagLeaf) ToSerializable() *SerializableDagLeaf {
156160
CurrentLinkCount: leaf.CurrentLinkCount,
157161
LatestLabel: leaf.LatestLabel,
158162
LeafCount: leaf.LeafCount,
163+
ContentSize: leaf.ContentSize,
164+
DagSize: leaf.DagSize,
159165
Links: make(map[string]string),
160166
AdditionalData: make(map[string]string),
161167
StoredProofs: make(map[string]*ClassicTreeBranch),
@@ -234,6 +240,8 @@ func TransmissionPacketFromSerializable(s *SerializableTransmissionPacket) *Tran
234240
CurrentLinkCount: s.Leaf.CurrentLinkCount,
235241
LatestLabel: s.Leaf.LatestLabel,
236242
LeafCount: s.Leaf.LeafCount,
243+
ContentSize: s.Leaf.ContentSize,
244+
DagSize: s.Leaf.DagSize,
237245
Links: make(map[string]string),
238246
AdditionalData: make(map[string]string),
239247
Proofs: make(map[string]*ClassicTreeBranch),

‎dag/types.go‎

Lines changed: 51 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -37,21 +37,57 @@ type DagBuilder struct {
3737
}
3838

3939
type DagLeaf struct {
40-
Hash string
41-
ItemName string
42-
Type LeafType
43-
ContentHash []byte
44-
Content []byte
45-
ClassicMerkleRoot []byte
46-
CurrentLinkCount int
47-
LatestLabel string
48-
LeafCount int
49-
Links map[string]string
50-
ParentHash string
51-
AdditionalData map[string]string
52-
MerkleTree *merkletree.MerkleTree
53-
LeafMap map[string]merkletree.DataBlock
54-
Proofs map[string]*ClassicTreeBranch
40+
// Hash is the CIDv1 content identifier for this leaf (label:hash format for non-root leaves)
41+
Hash string `json:"hash"`
42+
43+
// ItemName is the filename or directory name for this leaf
44+
ItemName string `json:"item_name"`
45+
46+
// Type specifies whether this is a file, directory, or chunk leaf
47+
Type LeafType `json:"type"`
48+
49+
// ContentHash is the SHA-256 hash of the Content field (only present if Content is set)
50+
ContentHash []byte `json:"content_hash,omitempty"`
51+
52+
// Content holds the actual file/chunk data
53+
Content []byte `json:"content,omitempty"`
54+
55+
// ClassicMerkleRoot is the merkle root of all child leaf hashes (empty if no children)
56+
ClassicMerkleRoot []byte `json:"classic_merkle_root,omitempty"`
57+
58+
// CurrentLinkCount is the total number of children this leaf has
59+
CurrentLinkCount int `json:"current_link_count"`
60+
61+
// LatestLabel is the highest numeric label among all leaves (only set on root leaf)
62+
LatestLabel string `json:"latest_label,omitempty"`
63+
64+
// LeafCount is the total number of leaves in the entire DAG (only set on root leaf)
65+
LeafCount int `json:"leaf_count,omitempty"`
66+
67+
// ContentSize is the total size of actual content data across the entire DAG in bytes (only set on root leaf)
68+
ContentSize int64 `json:"content_size,omitempty"`
69+
70+
// DagSize is the total serialized size of all leaves in the DAG in bytes (only set on root leaf)
71+
// Note: This value is approximate (within a few bytes) due to the two-pass serialization approach
72+
DagSize int64 `json:"dag_size,omitempty"`
73+
74+
// Links maps child labels to their hashes (label -> "label:hash")
75+
Links map[string]string `json:"links,omitempty"`
76+
77+
// ParentHash is the hash of the parent leaf (used during transmission, not stored in final DAG)
78+
ParentHash string `json:"parent_hash,omitempty"`
79+
80+
// AdditionalData holds custom metadata key-value pairs added by LeafProcessor
81+
AdditionalData map[string]string `json:"additional_data,omitempty"`
82+
83+
// MerkleTree is the computed merkle tree for this leaf's children (not serialized, computed on demand)
84+
MerkleTree *merkletree.MerkleTree `json:"-"`
85+
86+
// LeafMap maps child hashes to their merkle tree data blocks (not serialized, computed on demand)
87+
LeafMap map[string]merkletree.DataBlock `json:"-"`
88+
89+
// Proofs contains merkle proofs for children during partial DAG transmission (not stored in final DAG)
90+
Proofs map[string]*ClassicTreeBranch `json:"proofs,omitempty"`
5591
}
5692

5793
type DagLeafBuilder struct {

0 commit comments

Comments
 (0)