fixed some bugs and added new commands

Author: domkowald

data/csv/bib_core/readme.txt

@@ -187,9 +187,10 @@ public double getTagDiversity(List<Map<Integer, Double>> tagEntities) {
 			return diversity;
 		}
 
-		List<Integer> predictionIDs = new ArrayList<Integer>(); 
+		List<String> predictionIDs = new ArrayList<String>(); 
 		for (String res : this.predictionData) {
-			predictionIDs.add(Integer.valueOf(res));
+			//predictionIDs.add(Integer.valueOf(res));
+			predictionIDs.add(res);
 		}
 		int k = predictionIDs.size();
 		for (int i = 0; i < k; i++) {

data/metrics/bib_core/readme.txt

@@ -8,6 +8,7 @@ public class TimeUtil {
     public static final int DAY = 3;
     public static final int FIFTEEN_DAYS = 4;
     public static final int MONTH = 5;
+    public static final int WEEK = 6;
 
     /**
      * get duration count for the dataset.
@@ -43,6 +44,9 @@ public static int getDurationAtGranularity(int duration, int granularityLevel){
             case MONTH:
                 time_count = duration / secondsInMonth;
                 break;
+            case WEEK:
+                time_count = duration / secondsInWeek;
+                break;
         }
         return time_count;
     }

data/results/bib_core/readme.txt

@@ -332,6 +332,9 @@ public Map<String, Integer> getUserMap() {
 	}
 
 	public int getCountLimit() {
+		if (this.countLimit == 0) {
+			return this.getBookmarks().size();
+		}
 		return this.countLimit;
 	}
 

src/common/PredictionData.java

@@ -87,7 +87,7 @@ public boolean readFile(String filename, int k, BookmarkReader wikiReader, Integ
 				} else {
 					List<String> realData = Arrays.asList(lineParts[1].split(", "));
 					List<String> predictionData = Arrays.asList(lineParts[2].split(", "));
-					if (predictionData.size() > 0) {
+					if (predictionData.size() > 0 && realData.size() > 0) {
 						PredictionData data = new PredictionData(userID, resID, realData, predictionData, k);
 						this.predictions.add(data);
 						this.predictionCount++;

src/common/TimeUtil.java

@@ -0,0 +1,92 @@
+package file.preprocessing;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class JKULFMProcessor {
+	
+	private static final String EVENTS_FILE = "./data/schedl/LFM-1b_LEs.txt";
+	
+	private static Set<String> getFilterUsers(String filterFile) throws Exception {
+		Set<String> filterUsers = new HashSet<String>();
+		InputStreamReader reader = new InputStreamReader(new FileInputStream(new File(filterFile)), "UTF8");
+		BufferedReader br = new BufferedReader(reader);
+		String line = null;
+		int i = 0;
+		while ((line = br.readLine()) != null) {
+			if (i > 0) {
+				String userID = line.substring(0, line.indexOf(','));
+				filterUsers.add(userID);
+			}
+			i++;
+		}
+		System.out.println("Number of users: " + (i - 1));
+		br.close();
+		reader.close();
+		
+		return filterUsers;
+	}
+	
+	private static List<String> getFilterLines(Set<String> filterUsers) throws Exception {
+		String filePath = EVENTS_FILE;
+		List<String> filterLines = new ArrayList<String>();
+		
+		InputStreamReader reader = new InputStreamReader(new FileInputStream(new File(filePath)), "UTF8");
+		BufferedReader br = new BufferedReader(reader);
+		String line = null;
+		int i = 0;
+		while ((line = br.readLine()) != null) {
+			String userID = line.substring(0, line.indexOf('\t'));
+			if (filterUsers.contains(userID)) {
+				filterLines.add(line);
+				i++;
+			}
+		}
+		System.out.println("Number of lines: " + i);
+		br.close();
+		reader.close();
+		
+		return filterLines;
+	}
+	
+	private static void writeOutputFile(String outputFile, List<String> lines) throws Exception {
+		OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(new File(outputFile)), "UTF8");
+		BufferedWriter bw = new BufferedWriter(writer);
+		
+		int i = 0;
+		for(String l : lines) {
+			bw.write(l + '\n');
+			i++;
+		}
+		System.out.println("Written lines: " + i);
+		
+		bw.flush();
+		bw.close();
+		writer.close();
+	}
+	
+	public static void preprocessFile(String filterFile, String outputFile) {		
+		try {
+			// get filter users
+			Set<String> filterUsers = getFilterUsers(filterFile);
+			
+			// read big file and filter user actions
+			List<String> filterLines = getFilterLines(filterUsers);
+			
+			// write file
+			writeOutputFile(outputFile, filterLines);
+		} catch(Exception e) {
+			e.printStackTrace();
+		}
+	}
+}

src/file/BookmarkReader.java

@@ -251,7 +251,7 @@ public static double getLAverage(List<Map<Integer, Integer>> neighborMaps) {
 
 	private static String timeString;
 
-	private static List<Map<Integer, Double>> startBM25CreationForTagPrediction(BookmarkReader reader, int sampleSize, boolean userBased, boolean resBased, int beta) {
+	private static List<Map<Integer, Double>> startBM25CreationForTagPrediction(BookmarkReader reader, int sampleSize, boolean userBased, boolean resBased, int beta, boolean ignoreResource) {
 		int size = reader.getBookmarks().size();
 		int trainSize = size - sampleSize;
 		Stopwatch timer = new Stopwatch();
@@ -266,7 +266,8 @@ private static List<Map<Integer, Double>> startBM25CreationForTagPrediction(Book
 		for (int i = trainSize; i < size; i++) {
 			Bookmark data = reader.getBookmarks().get(i);
 			Map<Integer, Double> map = null;
-			map = calculator.getRankedTagList(data.getUserID(), data.getResourceID(), true);
+			int resID = (ignoreResource ? -1 : data.getResourceID());
+			map = calculator.getRankedTagList(data.getUserID(), resID, true);
 			results.add(map);
 			//System.out.println(data.getTags() + "|" + map.keySet());
 		}
@@ -277,12 +278,12 @@ private static List<Map<Integer, Double>> startBM25CreationForTagPrediction(Book
 		return results;
 	}	
 
-	public static BookmarkReader predictTags(String filename, int trainSize, int sampleSize, int neighbors, boolean userBased, boolean resBased, int beta) {
+	public static BookmarkReader predictTags(String filename, int trainSize, int sampleSize, int neighbors, boolean userBased, boolean resBased, int beta, boolean ignoreResource) {
 		MAX_NEIGHBORS = neighbors;
-		return predictSample(filename, trainSize, sampleSize, userBased, resBased, beta);
+		return predictSample(filename, trainSize, sampleSize, userBased, resBased, beta, ignoreResource);
 	}
 
-	public static BookmarkReader predictSample(String filename, int trainSize, int sampleSize, boolean userBased, boolean resBased, int beta) {
+	public static BookmarkReader predictSample(String filename, int trainSize, int sampleSize, boolean userBased, boolean resBased, int beta, boolean ignoreResource) {
 		Timer timerThread = new Timer();
 		MemoryThread memoryThread = new MemoryThread();
 		timerThread.schedule(memoryThread, 0, MemoryThread.TIME_SPAN);
@@ -291,7 +292,7 @@ public static BookmarkReader predictSample(String filename, int trainSize, int s
 		reader.readFile(filename);
 
 		List<Map<Integer, Double>> cfValues = null;	
-		cfValues = startBM25CreationForTagPrediction(reader, sampleSize, userBased, resBased, beta);
+		cfValues = startBM25CreationForTagPrediction(reader, sampleSize, userBased, resBased, beta, ignoreResource);
 
 		List<int[]> predictionValues = new ArrayList<int[]>();
 		for (int i = 0; i < cfValues.size(); i++) {