Skip to content
This repository was archived by the owner on Dec 15, 2025. It is now read-only.

Commit 3a6ec6d

Browse files
committed
Update Code to be prepare for hadoop version 3.2.0
* Created method copyMerge in TestDFSIOEnh.java due hadoop-common-3.2.0 jar has deprecatted it. [Removed FileUtil.copyMerge](https://issues.apache.org/jira/browse/HADOOP-12967). * Created method checkDest in TestDFSIOEnh.java because is private in org.apache.hadoop.fs.FileUtil and required for copyMerge. * LoggerFactory changed to org.slf4j due org.slf4j.Logger cannot be converted to org.apache.commons.logging.Log Signed-off-by: Luis Ponce <luis.f.ponce.navarro@linux.intel.com>
1 parent 19cbe05 commit 3a6ec6d

2 files changed

Lines changed: 64 additions & 9 deletions

File tree

autogen/src/main/java/org/apache/hadoop/fs/dfsioe/TestDFSIO.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
import java.util.Date;
2424
import java.util.StringTokenizer;
2525

26-
import org.apache.commons.logging.*;
26+
//import org.apache.commons.logging.*;
2727

2828
import org.apache.hadoop.fs.*;
2929
import org.apache.hadoop.mapred.*;
@@ -33,6 +33,8 @@
3333
import org.apache.hadoop.conf.*;
3434
import org.apache.hadoop.util.Tool;
3535
import org.apache.hadoop.util.ToolRunner;
36+
import org.slf4j.Logger;
37+
import org.slf4j.LoggerFactory;
3638

3739
/**
3840
* Distributed i/o benchmark.
@@ -69,8 +71,7 @@ public class TestDFSIO extends Configured implements Tool {
6971
private static final int DEFAULT_BUFFER_SIZE = 1000000;
7072
private static final String BASE_FILE_NAME = "test_io_";
7173
private static final String DEFAULT_RES_FILE_NAME = "TestDFSIO_results.log";
72-
73-
private static final Log LOG = FileInputFormat.LOG;
74+
private static final Logger LOG = LoggerFactory.getLogger(FileInputFormat.class);
7475
private static Configuration fsConfig = new Configuration();
7576
private static final long MEGA = 0x100000;
7677
private static String TEST_ROOT_DIR = System.getProperty("test.build.data","/benchmarks/TestDFSIO");

autogen/src/main/java/org/apache/hadoop/fs/dfsioe/TestDFSIOEnh.java

Lines changed: 60 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,14 @@
2222

2323
import java.util.Date;
2424
import java.util.StringTokenizer;
25+
import java.util.Arrays;
2526
import java.util.ArrayList;
2627
import java.util.Collections;
2728
import java.util.Comparator;
2829

29-
import org.apache.commons.logging.*;
30+
//import org.apache.commons.logging.*;
31+
import org.slf4j.Logger;
32+
import org.slf4j.LoggerFactory;
3033

3134
import org.apache.hadoop.mapred.*;
3235
import org.apache.hadoop.mapreduce.Job;
@@ -84,8 +87,7 @@
8487
*/
8588

8689
public class TestDFSIOEnh extends Configured implements Tool {
87-
88-
private static final Log LOG = LogFactory.getLog(TestDFSIOEnh.class);
90+
private static final Logger LOG = LoggerFactory.getLogger(TestDFSIOEnh.class);
8991
private static final int TEST_TYPE_READ = 0;
9092
private static final int TEST_TYPE_WRITE = 1;
9193
private static final int TEST_TYPE_CLEANUP = 2;
@@ -952,7 +954,7 @@ protected static void runAnalyse(FileSystem fs, Configuration fsConfig,
952954
e.printStackTrace();
953955
} finally {
954956
fs.delete(DfsioeConfig.getInstance().getReportTmp(fsConfig), true);
955-
FileUtil.copyMerge(fs, DfsioeConfig.getInstance().getReportDir(fsConfig), fs, DfsioeConfig.getInstance().getReportTmp(fsConfig), false, fsConfig, null);
957+
copyMerge(fs, DfsioeConfig.getInstance().getReportDir(fsConfig), fs, DfsioeConfig.getInstance().getReportTmp(fsConfig), false, fsConfig, null);
956958
LOG.info("remote report file " + DfsioeConfig.getInstance().getReportTmp(fsConfig) + " merged.");
957959
BufferedReader lines = new BufferedReader(new InputStreamReader(new DataInputStream(fs.open(DfsioeConfig.getInstance().getReportTmp(fsConfig)))));
958960
String line = null;
@@ -1001,8 +1003,60 @@ else if (sampleUnit == GIGA)
10011003
}
10021004
res.println("\n-- Result Analyse -- : " + ((System.currentTimeMillis() - t1)/1000) + "s");
10031005
res.close();
1004-
}
1005-
1006+
}
1007+
1008+
/** Copy all files in a directory to one output file (merge). */
1009+
@Deprecated
1010+
public static boolean copyMerge(FileSystem srcFS, Path srcDir, FileSystem dstFS, Path dstFile, boolean deleteSource,
1011+
Configuration conf, String addString) throws IOException {
1012+
dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false);
1013+
1014+
if (!srcFS.getFileStatus(srcDir).isDirectory())
1015+
return false;
1016+
1017+
OutputStream out = dstFS.create(dstFile);
1018+
1019+
try {
1020+
FileStatus contents[] = srcFS.listStatus(srcDir);
1021+
Arrays.sort(contents);
1022+
for (int i = 0; i < contents.length; i++) {
1023+
if (contents[i].isFile()) {
1024+
InputStream in = srcFS.open(contents[i].getPath());
1025+
try {
1026+
IOUtils.copyBytes(in, out, conf, false);
1027+
if (addString != null)
1028+
out.write(addString.getBytes("UTF-8"));
1029+
1030+
} finally {
1031+
in.close();
1032+
}
1033+
}
1034+
}
1035+
} finally {
1036+
out.close();
1037+
}
1038+
1039+
if (deleteSource) {
1040+
return srcFS.delete(srcDir, true);
1041+
} else {
1042+
return true;
1043+
}
1044+
}
1045+
1046+
private static Path checkDest(String srcName, FileSystem dstFS, Path dst, boolean overwrite) throws IOException {
1047+
if (dstFS.exists(dst)) {
1048+
FileStatus sdst = dstFS.getFileStatus(dst);
1049+
if (sdst.isDirectory()) {
1050+
if (null == srcName) {
1051+
throw new IOException("Target " + dst + " is a directory");
1052+
}
1053+
return checkDest(null, dstFS, new Path(dst, srcName), overwrite);
1054+
} else if (!overwrite) {
1055+
throw new IOException("Target " + dst + " already exists");
1056+
}
1057+
}
1058+
return dst;
1059+
}
10061060
@Deprecated
10071061
protected static void analyzeResult( FileSystem fs,
10081062
int testType,

0 commit comments

Comments
 (0)