|
22 | 22 |
|
23 | 23 | import java.util.Date; |
24 | 24 | import java.util.StringTokenizer; |
| 25 | +import java.util.Arrays; |
25 | 26 | import java.util.ArrayList; |
26 | 27 | import java.util.Collections; |
27 | 28 | import java.util.Comparator; |
28 | 29 |
|
29 | | -import org.apache.commons.logging.*; |
| 30 | +//import org.apache.commons.logging.*; |
| 31 | +import org.slf4j.Logger; |
| 32 | +import org.slf4j.LoggerFactory; |
30 | 33 |
|
31 | 34 | import org.apache.hadoop.mapred.*; |
32 | 35 | import org.apache.hadoop.mapreduce.Job; |
|
84 | 87 | */ |
85 | 88 |
|
86 | 89 | public class TestDFSIOEnh extends Configured implements Tool { |
87 | | - |
88 | | - private static final Log LOG = LogFactory.getLog(TestDFSIOEnh.class); |
| 90 | + private static final Logger LOG = LoggerFactory.getLogger(TestDFSIOEnh.class); |
89 | 91 | private static final int TEST_TYPE_READ = 0; |
90 | 92 | private static final int TEST_TYPE_WRITE = 1; |
91 | 93 | private static final int TEST_TYPE_CLEANUP = 2; |
@@ -952,7 +954,7 @@ protected static void runAnalyse(FileSystem fs, Configuration fsConfig, |
952 | 954 | e.printStackTrace(); |
953 | 955 | } finally { |
954 | 956 | fs.delete(DfsioeConfig.getInstance().getReportTmp(fsConfig), true); |
955 | | - FileUtil.copyMerge(fs, DfsioeConfig.getInstance().getReportDir(fsConfig), fs, DfsioeConfig.getInstance().getReportTmp(fsConfig), false, fsConfig, null); |
| 957 | + copyMerge(fs, DfsioeConfig.getInstance().getReportDir(fsConfig), fs, DfsioeConfig.getInstance().getReportTmp(fsConfig), false, fsConfig, null); |
956 | 958 | LOG.info("remote report file " + DfsioeConfig.getInstance().getReportTmp(fsConfig) + " merged."); |
957 | 959 | BufferedReader lines = new BufferedReader(new InputStreamReader(new DataInputStream(fs.open(DfsioeConfig.getInstance().getReportTmp(fsConfig))))); |
958 | 960 | String line = null; |
@@ -1001,8 +1003,60 @@ else if (sampleUnit == GIGA) |
1001 | 1003 | } |
1002 | 1004 | res.println("\n-- Result Analyse -- : " + ((System.currentTimeMillis() - t1)/1000) + "s"); |
1003 | 1005 | res.close(); |
1004 | | - } |
1005 | | - |
| 1006 | + } |
| 1007 | + |
| 1008 | + /** Copy all files in a directory to one output file (merge). */ |
| 1009 | + @Deprecated |
| 1010 | + public static boolean copyMerge(FileSystem srcFS, Path srcDir, FileSystem dstFS, Path dstFile, boolean deleteSource, |
| 1011 | + Configuration conf, String addString) throws IOException { |
| 1012 | + dstFile = checkDest(srcDir.getName(), dstFS, dstFile, false); |
| 1013 | + |
| 1014 | + if (!srcFS.getFileStatus(srcDir).isDirectory()) |
| 1015 | + return false; |
| 1016 | + |
| 1017 | + OutputStream out = dstFS.create(dstFile); |
| 1018 | + |
| 1019 | + try { |
| 1020 | + FileStatus contents[] = srcFS.listStatus(srcDir); |
| 1021 | + Arrays.sort(contents); |
| 1022 | + for (int i = 0; i < contents.length; i++) { |
| 1023 | + if (contents[i].isFile()) { |
| 1024 | + InputStream in = srcFS.open(contents[i].getPath()); |
| 1025 | + try { |
| 1026 | + IOUtils.copyBytes(in, out, conf, false); |
| 1027 | + if (addString != null) |
| 1028 | + out.write(addString.getBytes("UTF-8")); |
| 1029 | + |
| 1030 | + } finally { |
| 1031 | + in.close(); |
| 1032 | + } |
| 1033 | + } |
| 1034 | + } |
| 1035 | + } finally { |
| 1036 | + out.close(); |
| 1037 | + } |
| 1038 | + |
| 1039 | + if (deleteSource) { |
| 1040 | + return srcFS.delete(srcDir, true); |
| 1041 | + } else { |
| 1042 | + return true; |
| 1043 | + } |
| 1044 | + } |
| 1045 | + |
| 1046 | + private static Path checkDest(String srcName, FileSystem dstFS, Path dst, boolean overwrite) throws IOException { |
| 1047 | + if (dstFS.exists(dst)) { |
| 1048 | + FileStatus sdst = dstFS.getFileStatus(dst); |
| 1049 | + if (sdst.isDirectory()) { |
| 1050 | + if (null == srcName) { |
| 1051 | + throw new IOException("Target " + dst + " is a directory"); |
| 1052 | + } |
| 1053 | + return checkDest(null, dstFS, new Path(dst, srcName), overwrite); |
| 1054 | + } else if (!overwrite) { |
| 1055 | + throw new IOException("Target " + dst + " already exists"); |
| 1056 | + } |
| 1057 | + } |
| 1058 | + return dst; |
| 1059 | + } |
1006 | 1060 | @Deprecated |
1007 | 1061 | protected static void analyzeResult( FileSystem fs, |
1008 | 1062 | int testType, |
|
0 commit comments