Skip to content

Commit 4e90420

Browse files
pbloempbloem
committed
Fixed bug in DiskDGraph loading
1 parent f80beb1 commit 4e90420

7 files changed

Lines changed: 88431 additions & 42 deletions

File tree

‎nodes/pom.xml‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -114,11 +114,12 @@
114114
<artifactId>mapdb</artifactId>
115115
<version>3.0.1</version>
116116
</dependency>
117-
<dependency>
118-
<groupId>com.github.pbloem</groupId>
119-
<artifactId>externalsortinginjava</artifactId>
120-
<version>v0.1.0</version>
121-
</dependency>
117+
118+
<dependency>
119+
<groupId>com.github.pbloem</groupId>
120+
<artifactId>externalsortinginjava</artifactId>
121+
<version>v0.1.0</version>
122+
</dependency>
122123
</dependencies>
123124

124125
<repositories>

‎nodes/src/main/java/org/nodes/DiskDGraph.java‎

Lines changed: 29 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,8 @@ public DiskDGraph(File dir)
9494
*/
9595
public DiskDGraph(File dir, boolean nullLabels)
9696
{
97+
this.nullLabels = nullLabels;
98+
9799
dir.mkdirs();
98100
File dbFile = new File(dir, "graph."+id+".db");
99101

@@ -523,11 +525,11 @@ public List<DLink<String>> links()
523525

524526
List<DLink<String>> list = new ArrayList<DLink<String>>(degree());
525527

526-
for(int neighbor : out.get(index))
528+
for(int neighbor : out.get((int)index))
527529
list.add(new DiskDLink(index, neighbor));
528530

529-
for(int neighbor : in.get(index))
530-
if(neighbor != index) // no double reflexive links
531+
for(int neighbor : in.get((int)index))
532+
if(neighbor != ((int)index)) // no double reflexive links
531533
list.add(new DiskDLink(neighbor, index));
532534

533535
return list;
@@ -539,7 +541,7 @@ public List<DLink<String>> linksOut()
539541
check();
540542

541543
List<DLink<String>> list = new ArrayList<DLink<String>>(outDegree());
542-
for(int neighbor : out.get(index))
544+
for(int neighbor : out.get((int)index))
543545
list.add(new DiskDLink(index, neighbor));
544546

545547
return list;
@@ -551,7 +553,7 @@ public List<DLink<String>> linksIn()
551553
check();
552554

553555
List<DLink<String>> list = new ArrayList<DLink<String>>(inDegree());
554-
for(int neighbor : in.get(index))
556+
for(int neighbor : in.get((int)index))
555557
list.add(new DiskDLink(neighbor, index));
556558

557559
return list;
@@ -565,12 +567,12 @@ public Collection<? extends DLink<String>> links(Node<String> other)
565567
List<DLink<String>> list = new ArrayList<DLink<String>>();
566568

567569
int o = other.index();
568-
for(int neighbor : out.get(index))
570+
for(int neighbor : out.get((int)index))
569571
if(neighbor == o)
570572
list.add(new DiskDLink(index, neighbor));
571573

572574
if(index != o)
573-
for(int neighbor : in.get(index))
575+
for(int neighbor : in.get((int)index))
574576
if(neighbor == o)
575577
list.add(new DiskDLink(neighbor, index));
576578

@@ -585,8 +587,8 @@ public Collection<? extends DLink<String>> linksOut(DNode<String> other)
585587
List<DLink<String>> list = new ArrayList<DLink<String>>(outDegree());
586588

587589
int o = other.index();
588-
for(int neighbor : out.get(index))
589-
if(neighbor == o)
590+
for(int neighbor : out.get((int)index))
591+
if(((int)neighbor) == ((int)o))
590592
list.add(new DiskDLink(index, neighbor));
591593

592594
return list;
@@ -600,7 +602,7 @@ public Collection<? extends DLink<String>> linksIn(DNode<String> other)
600602
List<DLink<String>> list = new ArrayList<DLink<String>>(inDegree());
601603

602604
int o = other.index();
603-
for(int neighbor : in.get(index))
605+
for(int neighbor : in.get((int)index))
604606
if(neighbor == o)
605607
list.add(new DiskDLink(neighbor, index));
606608

@@ -612,7 +614,6 @@ private class DiskDLink implements DLink<String>
612614
{
613615
private DNode<String> from, to;
614616

615-
616617
private long nodeModState = nodeModCount;
617618

618619
private boolean dead = false;
@@ -901,7 +902,7 @@ private void read()
901902
public DNode<String> add(String label)
902903
{
903904

904-
if(!nullLabels && label != null)
905+
if(nullLabels && label != null)
905906
throw new IllegalArgumentException("Graph is set to null labels only.");
906907

907908
if(! nullLabels)
@@ -1169,11 +1170,12 @@ public static DiskDGraph fromFile(File file, File dir)
11691170
// * sort the input file by first element
11701171
File forward = new File(dir, "forward.edgelist");
11711172

1172-
ExternalSort.mergeSortedFiles(
1173-
ExternalSort.sortInBatch(
1173+
1174+
List<File> files = ExternalSort.sortInBatch(
11741175
file,
11751176
new LComp(true), ExternalSort.DEFAULTMAXTEMPFILES,
1176-
Charset.defaultCharset(), dir, false), forward);
1177+
Charset.defaultCharset(), dir, false);
1178+
ExternalSort.mergeSortedFiles(files, forward, new LComp(true), Charset.defaultCharset());
11771179

11781180
System.out.println("Forward sort finished");
11791181

@@ -1184,11 +1186,11 @@ public static DiskDGraph fromFile(File file, File dir)
11841186
forward.delete();
11851187
File backward = new File(dir, "backward.edgelist");
11861188

1187-
ExternalSort.mergeSortedFiles(
1188-
ExternalSort.sortInBatch(
1189+
files = ExternalSort.sortInBatch(
11891190
file,
11901191
new LComp(false), ExternalSort.DEFAULTMAXTEMPFILES,
1191-
Charset.defaultCharset(), dir, false), backward);
1192+
Charset.defaultCharset(), dir, false);
1193+
ExternalSort.mergeSortedFiles(files, backward, new LComp(false), Charset.defaultCharset());
11921194

11931195
System.out.println("Backward sort finished");
11941196

@@ -1272,9 +1274,17 @@ private static long readSorted(List<List<Integer>> list, File file, boolean forw
12721274

12731275
if(a != (int) current)
12741276
{
1275-
list.add(neighbors);
1277+
try {
1278+
list.add(neighbors);
1279+
} catch(AssertionError e)
1280+
{
1281+
throw new AssertionError("Failed to add list to IndexTreeList. current list size: "+list.size()+", list to be added "+neighbors);
1282+
}
12761283
neighbors.clear();
12771284

1285+
if(a < list.size())
1286+
throw new IllegalStateException("Next index is "+a+", while list size is already " + list.size() + ". It seems like the sorting ot the file went wrong.");
1287+
12781288
while(list.size() < a)
12791289
list.add(Collections.EMPTY_LIST);
12801290

‎nodes/src/main/java/org/nodes/data/Examples.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,4 +148,24 @@ public static UGraph<String> citations()
148148
throw new RuntimeException("Could not load the file for the citations graph from the classpath.", e);
149149
}
150150
}
151+
152+
/**
153+
* A snapshot of part of the Gnutella P2P network. Source:
154+
* http://snap.stanford.edu/data/p2p-Gnutella30.html
155+
*
156+
* @return
157+
*/
158+
public static DGraph<String> p2p()
159+
{
160+
ClassLoader classLoader = Examples.class.getClassLoader();
161+
File file = new File(classLoader.getResource("graphs/p2p/p2p.txt").getFile());
162+
163+
try
164+
{
165+
return Data.edgeListDirected(file);
166+
} catch (IOException e)
167+
{
168+
throw new RuntimeException("Could not load the file for the P2P graph from the classpath.", e);
169+
}
170+
}
151171
}

‎nodes/src/main/java/org/nodes/random/SimpleSubgraphGenerator.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ public List<Integer> generate()
6363

6464
result.clear();
6565
result.add(Global.random().nextInt(graph.size()));
66+
6667
restarts ++;
6768
if(restarts > RESTARTS)
6869
checkLCC(depth);

‎nodes/src/test/java/org/nodes/DiskDGraphTest.java‎

Lines changed: 42 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.junit.Test;
1919
import org.nodes.data.Data;
2020
import org.nodes.data.Examples;
21+
import org.nodes.random.RandomGraphs;
2122
import org.omg.Messaging.SyncScopeHelper;
2223

2324
import nl.peterbloem.kit.FileIO;
@@ -279,34 +280,38 @@ public void testImportBig()
279280
System.out.println(graph.numLinks());
280281
}
281282

283+
284+
/**
285+
* Run with low heap space...
286+
*
287+
* @throws IOException
288+
*/
282289
@Test
283290
public void testImport()
284291
throws IOException
285292
{
286293
Global.randomSeed();
287294

288-
FileIO.copy("graphs/citations/citations.txt", DIR);
295+
FileIO.copy("graphs/p2p/p2p.txt", DIR);
289296

290-
DGraph<String> diskGraph = DiskDGraph.fromFile(new File(DIR, "citations.txt"), DIR);
297+
DGraph<String> diskGraph = DiskDGraph.fromFile(new File(DIR, "p2p.txt"), DIR);
291298
assertEquals(diskGraph.size(), new ArrayList<DNode<String>>(diskGraph.nodes()).size());
292299
assertEquals(diskGraph.numLinks(), new ArrayList<DLink<String>>(diskGraph.links()).size());
293300

294-
DGraph<String> memGraph = Data.edgeListDirectedUnlabeled(new File(DIR, "citations.txt"), true);
295-
296-
// diskGraph = LightDGraph.copy(diskGraph);
297-
298-
for(int i : series(memGraph.size()))
299-
{
300-
String a = diskGraph.get(i).out() + " " + diskGraph.get(i).in();
301-
String b = memGraph.get(i).out() + " " + memGraph.get(i).in();
302-
303-
if(! a.equals(b))
304-
{
305-
System.out.println("d " + a);
306-
System.out.println("m " + b);
307-
}
301+
DGraph<String> memGraph = Data.edgeListDirectedUnlabeled(new File(DIR, "p2p.txt"), true);
308302

309-
}
303+
// for(int i : series(memGraph.size()))
304+
// {
305+
// String a = diskGraph.get(i).out() + " " + diskGraph.get(i).in();
306+
//
307+
// String b = memGraph.get(i).out() + " " + memGraph.get(i).in();
308+
//
309+
// if(! a.equals(b))
310+
// {
311+
// System.out.println("d " + a);
312+
// System.out.println("m " + b);
313+
// }
314+
// }
310315

311316
assertEquals(memGraph, diskGraph);
312317
}
@@ -578,7 +583,26 @@ public void testNeighborsFast()
578583
}
579584
}
580585

581-
@After
586+
/**
587+
* Test subgraph extraction
588+
*
589+
*/
590+
@Test
591+
public void testJBC()
592+
{
593+
DGraph<String> graph = Graphs.jbcDirected();
594+
graph = DiskDGraph.copy(graph, DIR);
595+
596+
List<Integer> nodes = Arrays.asList(13, 15, 16);
597+
598+
DGraph<String> subgraph = Subgraph.dSubgraphIndices(graph, nodes);
599+
System.out.println(subgraph);
600+
601+
assertEquals(3, subgraph.size());
602+
assertEquals(2, subgraph.numLinks());
603+
}
604+
605+
// @After
582606
public void cleanup()
583607
{
584608
for(File file : DIR.listFiles())

‎nodes/src/test/resources/graphs/p2p/metadata.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
http://snap.stanford.edu/data/p2p-Gnutella30.html

0 commit comments

Comments
 (0)