Skip to content
This repository was archived by the owner on Dec 15, 2025. It is now read-only.

Commit 83992e1

Browse files
harschwarecarsonwang
harschware
authored and
carsonwang
committed
Fixes an issue where DFSioe ignores the value of test.build.data (#276)
* Fixes an issue where DFSioe ignores the value of test.build.data property * fixes cannot find symbol TEST_ROOT_DIR
1 parent d076a43 commit 83992e1

4 files changed

Lines changed: 180 additions & 31 deletions

File tree

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,6 @@ target/
1111
derby.log
1212
metastore_db/
1313
report/
14+
.classpath
15+
.project
16+
.settings/

‎src/autogen/src/main/java/org/apache/hadoop/fs/dfsioe/Analyzer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ public static void main(String[] args) throws Exception{
107107
job.setReducerClass(_Reducer.class);
108108
job.setOutputKeyClass(Text.class);
109109
job.setOutputValueClass(Text.class);
110-
FileInputFormat.addInputPath(job, new Path(TestDFSIOEnh.READ_DIR, "part-00000"));
110+
FileInputFormat.addInputPath(job, new Path(DfsioeConfig.getInstance().getReadDir(conf), "part-00000"));
111111
FileOutputFormat.setOutputPath(job, outdir);
112112
System.exit(job.waitForCompletion(true) ? 0 : 1);
113113
}

‎src/autogen/src/main/java/org/apache/hadoop/fs/dfsioe/DfsioeConfig.java‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
package org.apache.hadoop.fs.dfsioe;
2+
3+
import org.apache.commons.logging.Log;
4+
import org.apache.commons.logging.LogFactory;
5+
import org.apache.hadoop.conf.Configuration;
6+
import org.apache.hadoop.fs.Path;
7+
8+
public final class DfsioeConfig {
9+
private static DfsioeConfig instance = null;
10+
11+
private DfsioeConfig() {
12+
}
13+
14+
public static synchronized DfsioeConfig getInstance() {
15+
if (instance == null)
16+
instance = new DfsioeConfig();
17+
return instance;
18+
}
19+
20+
private static final Log LOG = LogFactory.getLog(DfsioeConfig.class);
21+
22+
private String testRootDir;
23+
private Path controlDir;
24+
private Path writeDir;
25+
private Path readDir;
26+
private Path dataDir;
27+
private Path reportDir;
28+
private Path reportTmp;
29+
30+
public String getTestRootDir() {
31+
if (testRootDir == null) {
32+
throw new RuntimeException(
33+
"testRootDir should not be null at this point");
34+
}
35+
return testRootDir;
36+
}
37+
38+
public String getTestRootDir(Configuration conf) {
39+
String propName = "test.build.data";
40+
if (testRootDir == null) {
41+
testRootDir = System.getProperty(propName);
42+
if( testRootDir != null ) {
43+
LOG.info("Found '" + propName + "' in System properties, value is '" + testRootDir + "'");
44+
} else {
45+
testRootDir = safeGet(conf, propName);
46+
LOG.info("Found '" + propName + "' in Hadoop configuration, value is '" + testRootDir + "'");
47+
}
48+
LOG.info("Setting testRootDir to '" + testRootDir + "'");
49+
}
50+
51+
return testRootDir;
52+
}
53+
54+
public Path getControlDir() {
55+
if (controlDir == null) {
56+
controlDir = new Path(getTestRootDir(), "io_control");
57+
}
58+
return controlDir;
59+
}
60+
61+
public Path getControlDir(Configuration conf) {
62+
controlDir = new Path(getTestRootDir(conf), "io_control");
63+
64+
return controlDir;
65+
}
66+
67+
public Path getWriteDir() {
68+
if (writeDir == null) {
69+
writeDir = new Path(getTestRootDir(), "io_write");
70+
}
71+
return writeDir;
72+
}
73+
74+
public Path getWriteDir(Configuration conf) {
75+
writeDir = new Path(getTestRootDir(conf), "io_write");
76+
77+
return writeDir;
78+
}
79+
80+
public Path getReadDir() {
81+
if (readDir == null) {
82+
readDir = new Path(getTestRootDir(), "io_read");
83+
}
84+
return readDir;
85+
}
86+
87+
public Path getReadDir(Configuration conf) {
88+
readDir = new Path(getTestRootDir(conf), "io_read");
89+
90+
return readDir;
91+
}
92+
93+
public Path getDataDir() {
94+
if (dataDir == null) {
95+
dataDir = new Path(getTestRootDir(), "io_data");
96+
}
97+
return dataDir;
98+
}
99+
100+
public Path getDataDir(Configuration conf) {
101+
dataDir = new Path(getTestRootDir(conf), "io_data");
102+
103+
return dataDir;
104+
}
105+
106+
public Path getReportDir() {
107+
if (reportDir == null) {
108+
reportDir = new Path(getTestRootDir(), "reports");
109+
}
110+
return reportDir;
111+
}
112+
113+
public Path getReportDir(Configuration conf) {
114+
reportDir = new Path(getTestRootDir(conf), "reports");
115+
116+
return reportDir;
117+
}
118+
119+
public Path getReportTmp() {
120+
if (reportTmp == null) {
121+
reportTmp = new Path(getTestRootDir(), "_merged_reports.txt");
122+
}
123+
return reportTmp;
124+
}
125+
126+
public Path getReportTmp(Configuration conf) {
127+
reportTmp = new Path(getTestRootDir(conf), "_merged_reports.txt");
128+
129+
return reportTmp;
130+
}
131+
132+
public String safeGet(Configuration conf, String prop) {
133+
if (conf == null) {
134+
throw new RuntimeException("null config not allowed");
135+
}
136+
if (conf.get(prop) == null) {
137+
throw new RuntimeException("Property " + prop + " cannot be empty");
138+
}
139+
return conf.get(prop);
140+
}
141+
142+
}

‎src/autogen/src/main/java/org/apache/hadoop/fs/dfsioe/TestDFSIOEnh.java‎

Lines changed: 34 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -93,14 +93,18 @@ public class TestDFSIOEnh extends Configured implements Tool {
9393
private static final String BASE_FILE_NAME = "test_io_";
9494
private static final String DEFAULT_RES_FILE_NAME = "TestDFSIOEnh_results.log";
9595

96-
private static String TEST_ROOT_DIR = System.getProperty("test.build.data","/benchmarks/TestDFSIO-Enh");
9796
private static Configuration fsConfig = new Configuration();
98-
private static Path CONTROL_DIR = new Path(TEST_ROOT_DIR, "io_control");
99-
protected static Path WRITE_DIR = new Path(TEST_ROOT_DIR, "io_write");
100-
protected static Path READ_DIR = new Path(TEST_ROOT_DIR, "io_read");
101-
private static Path DATA_DIR = new Path(TEST_ROOT_DIR, "io_data");
102-
private static Path REPORT_DIR = new Path(TEST_ROOT_DIR, "reports");
103-
private static Path REPORT_TMP = new Path(TEST_ROOT_DIR, "_merged_reports.txt");
97+
98+
// NOTE: Access to these static variables have been replaced with a singleton config class, which gets the TEST_ROOT_DIR
99+
// from the job configuration (since it is not accessible via system properties)
100+
//
101+
// private static String TEST_ROOT_DIR = System.getProperty("test.build.data","/benchmarks/TestDFSIO-Enh");
102+
// private static Path CONTROL_DIR = new Path(TEST_ROOT_DIR, "io_control");
103+
// protected static Path WRITE_DIR = new Path(TEST_ROOT_DIR, "io_write");
104+
// protected static Path READ_DIR = new Path(TEST_ROOT_DIR, "io_read");
105+
// private static Path DATA_DIR = new Path(TEST_ROOT_DIR, "io_data");
106+
// private static Path REPORT_DIR = new Path(TEST_ROOT_DIR, "reports");
107+
// private static Path REPORT_TMP = new Path(TEST_ROOT_DIR, "_merged_reports.txt");
104108

105109

106110
static{
@@ -206,7 +210,7 @@ public Object doIO(Reporter reporter,
206210

207211
totalSize *= MEGA;
208212
OutputStream out;
209-
out = fs.create(new Path(DATA_DIR, name), true, bufferSize);
213+
out = fs.create(new Path(DfsioeConfig.getInstance().getDataDir(getConf()), name), true, bufferSize);
210214
stats.objSize = totalSize;
211215

212216
long lastTimeStamp=0;
@@ -257,15 +261,15 @@ private static String getFileName(int fIdx) {
257261
protected static void writeTest(FileSystem fs, Configuration fsConfig)
258262
throws IOException {
259263

260-
fs.delete(DATA_DIR, true);
261-
fs.delete(WRITE_DIR, true);
264+
fs.delete(DfsioeConfig.getInstance().getDataDir(fsConfig), true);
265+
fs.delete(DfsioeConfig.getInstance().getWriteDir(fsConfig), true);
262266

263267
//set write specific configurations
264268
JobConf job = new JobConf(fsConfig, TestDFSIOEnh.class);
265269
//should turn off speculative execution to avoid two attemps writing to the same file
266270
job.setMapSpeculativeExecution(false);
267271

268-
runIOTest(WriteMapperEnh.class, WRITE_DIR,job);
272+
runIOTest(WriteMapperEnh.class, DfsioeConfig.getInstance().getWriteDir(fsConfig),job);
269273
}
270274

271275

@@ -286,7 +290,7 @@ public Object doIO(Reporter reporter,
286290

287291
totalSize *= MEGA;
288292
// open file
289-
DataInputStream in = fs.open(new Path(DATA_DIR, name));
293+
DataInputStream in = fs.open(new Path(DfsioeConfig.getInstance().getDataDir(fsConfig), name));
290294
stats.objSize = totalSize;
291295

292296
long lastTimeStamp = 0;
@@ -331,12 +335,12 @@ public Object doIO(Reporter reporter,
331335
}
332336

333337
protected static void readTest(FileSystem fs,Configuration fsConfig) throws IOException {
334-
fs.delete(READ_DIR, true);
338+
fs.delete(DfsioeConfig.getInstance().getReadDir(fsConfig), true);
335339
//set read job specific configurations
336340
JobConf job = new JobConf(fsConfig, TestDFSIOEnh.class);
337341
//turn on speculative execution to potentially improve performance.
338342
//job.setMapSpeculativeExecution(true);
339-
runIOTest(ReadMapperEnh.class, READ_DIR,job);
343+
runIOTest(ReadMapperEnh.class, DfsioeConfig.getInstance().getReadDir(fsConfig),job);
340344
}
341345

342346
protected static void sequentialTest(
@@ -362,7 +366,7 @@ protected static void runIOTest( @SuppressWarnings("rawtypes") Class<? extends M
362366
Path outputDir,
363367
JobConf job
364368
) throws IOException {
365-
FileInputFormat.setInputPaths(job, CONTROL_DIR);
369+
FileInputFormat.setInputPaths(job, DfsioeConfig.getInstance().getControlDir(fsConfig));
366370
job.setInputFormat(SequenceFileInputFormat.class);
367371

368372
job.setMapperClass(mapperClass);
@@ -571,7 +575,7 @@ else if (unit.equalsIgnoreCase("g"))
571575
fsConfig.setInt("test.io.file.buffer.size", bufferSize);
572576
fsConfig.setInt("test.io.sampling.interval",tputSampleInterval);
573577

574-
FileSystem fs = FileSystem.get(new Path(TEST_ROOT_DIR).toUri(), fsConfig);
578+
FileSystem fs = FileSystem.get(fsConfig);
575579

576580
//get the configuration of max number of concurrent maps
577581
JobConf dummyConf = new JobConf(fsConfig, TestDFSIOEnh.class);
@@ -633,11 +637,11 @@ private static void createControlFile(
633637
) throws IOException {
634638
LOG.info("creating control file: "+fileSize+" mega bytes, "+nrFiles+" files");
635639

636-
fs.delete(CONTROL_DIR, true);
640+
fs.delete(DfsioeConfig.getInstance().getControlDir(fsConfig), true);
637641

638642
for(int i=0; i < nrFiles; i++) {
639643
String name = getFileName(i);
640-
Path controlFile = new Path(CONTROL_DIR, "in_file_" + name);
644+
Path controlFile = new Path(DfsioeConfig.getInstance().getControlDir(fsConfig), "in_file_" + name);
641645
SequenceFile.Writer writer = null;
642646
try {
643647
writer = SequenceFile.createWriter(fs, fsConfig, controlFile,
@@ -838,9 +842,9 @@ protected static void runAnalyse(FileSystem fs, Configuration fsConfig,
838842
long t1 = System.currentTimeMillis();
839843
Path reduceFile;
840844
if (testType == TEST_TYPE_WRITE)
841-
reduceFile = new Path(WRITE_DIR, "part-00000");
845+
reduceFile = new Path(DfsioeConfig.getInstance().getWriteDir(fsConfig), "part-00000");
842846
else
843-
reduceFile = new Path(READ_DIR, "part-00000");
847+
reduceFile = new Path(DfsioeConfig.getInstance().getReadDir(fsConfig), "part-00000");
844848

845849
int maxslot = (int)(execTime/plotInterval)+1;
846850
int[] concurrency = new int[maxslot+1];
@@ -923,7 +927,7 @@ protected static void runAnalyse(FileSystem fs, Configuration fsConfig,
923927
}
924928

925929
try {
926-
fs.delete(REPORT_DIR, true);
930+
fs.delete(DfsioeConfig.getInstance().getReportDir(fsConfig), true);
927931
//set up env
928932
Configuration conf2 = new Configuration(fsConfig);
929933
conf2.setLong("ana_tStart", tStart);
@@ -940,17 +944,17 @@ protected static void runAnalyse(FileSystem fs, Configuration fsConfig,
940944
job.setOutputValueClass(Text.class);
941945
// job.setNumReduceTasks(1);
942946
org.apache.hadoop.mapreduce.lib.input.FileInputFormat.addInputPath(job, reduceFile);
943-
org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.setOutputPath(job, REPORT_DIR);
947+
org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.setOutputPath(job, DfsioeConfig.getInstance().getReportDir(fsConfig));
944948
job.waitForCompletion(true);
945949
} catch (InterruptedException e) {
946950
e.printStackTrace();
947951
} catch (ClassNotFoundException e) {
948952
e.printStackTrace();
949953
} finally {
950-
fs.delete(REPORT_TMP, true);
951-
FileUtil.copyMerge(fs, REPORT_DIR, fs, REPORT_TMP, false, fsConfig, null);
952-
LOG.info("remote report file " + REPORT_TMP + " merged.");
953-
BufferedReader lines = new BufferedReader(new InputStreamReader(new DataInputStream(fs.open(REPORT_TMP))));
954+
fs.delete(DfsioeConfig.getInstance().getReportTmp(fsConfig), true);
955+
FileUtil.copyMerge(fs, DfsioeConfig.getInstance().getReportDir(fsConfig), fs, DfsioeConfig.getInstance().getReportTmp(fsConfig), false, fsConfig, null);
956+
LOG.info("remote report file " + DfsioeConfig.getInstance().getReportTmp(fsConfig) + " merged.");
957+
BufferedReader lines = new BufferedReader(new InputStreamReader(new DataInputStream(fs.open(DfsioeConfig.getInstance().getReportTmp(fsConfig)))));
954958
String line = null;
955959
while((line = lines.readLine()) != null) {
956960
StringTokenizer tokens = new StringTokenizer(line, " \t\n\r\f%");
@@ -964,7 +968,7 @@ protected static void runAnalyse(FileSystem fs, Configuration fsConfig,
964968
}
965969
lines.close();
966970
if (tputReportEach) {
967-
FileUtil.copy(fs, REPORT_TMP, new File(tputResFileName.split(".")[0]+"test_io_.csv"), false, fsConfig);
971+
FileUtil.copy(fs, DfsioeConfig.getInstance().getReportTmp(fsConfig), new File(tputResFileName.split(".")[0]+"test_io_.csv"), false, fsConfig);
968972
LOG.info("*test_io_.csv fetched to local fs.");
969973
}
970974
//calculate the aggregated throughput
@@ -1028,9 +1032,9 @@ protected static void analyzeResult( FileSystem fs,
10281032

10291033
Path reduceFile;
10301034
if (testType == TEST_TYPE_WRITE)
1031-
reduceFile = new Path(WRITE_DIR, "part-00000");
1035+
reduceFile = new Path(DfsioeConfig.getInstance().getWriteDir(fsConfig), "part-00000");
10321036
else
1033-
reduceFile = new Path(READ_DIR, "part-00000");
1037+
reduceFile = new Path(DfsioeConfig.getInstance().getReadDir(fsConfig), "part-00000");
10341038

10351039

10361040
//long time = 0;
@@ -1132,7 +1136,7 @@ protected static void analyzeResult( FileSystem fs,
11321136

11331137
private static void cleanup(FileSystem fs) throws IOException {
11341138
LOG.info("Cleaning up test files");
1135-
fs.delete(new Path(TEST_ROOT_DIR), true);
1139+
fs.delete(new Path(DfsioeConfig.getInstance().getTestRootDir(fsConfig)), true);
11361140
}
11371141

11381142
}//end

0 commit comments

Comments
 (0)