001package es.ucm.fdi.gaia.jcolibri.evaluation.evaluators; 002 003import org.apache.logging.log4j.LogManager; 004 005import es.ucm.fdi.gaia.jcolibri.casebase.CachedLinealCaseBase; 006import es.ucm.fdi.gaia.jcolibri.cbraplications.StandardCBRApplication; 007import es.ucm.fdi.gaia.jcolibri.cbrcore.CBRCase; 008import es.ucm.fdi.gaia.jcolibri.cbrcore.CBRCaseBase; 009import es.ucm.fdi.gaia.jcolibri.evaluation.EvaluationReport; 010import es.ucm.fdi.gaia.jcolibri.evaluation.Evaluator; 011import es.ucm.fdi.gaia.jcolibri.exception.ExecutionException; 012import es.ucm.fdi.gaia.jcolibri.util.ProgressController; 013 014import java.util.ArrayList; 015import java.util.Collection; 016import java.util.Date; 017import java.util.List; 018 019/** 020 * This method splits the case base in two sets: one used for testing where each 021 * case is used as query, and another that acts as normal case base. This 022 * process is performed serveral times. 023 * 024 * @author Juan A. Recio GarcĂa - GAIA http://gaia.fdi.ucm.es 025 * @version 2.0 026 */ 027public class HoldOutEvaluator extends Evaluator { 028 029 protected StandardCBRApplication app; 030 031 public void init(StandardCBRApplication cbrApp) { 032 033 report = new EvaluationReport(); 034 app = cbrApp; 035 try { 036 app.configure(); 037 } catch (ExecutionException e) { 038 LogManager.getLogger().error(e); 039 } 040 } 041 042 /** 043 * Performs the Hold-Out evaluation. 044 * @param testPercent Percent of the case base used as queries. The case base is splited randomly in each repetition. 045 * @param repetitions Number of repetitions. 046 */ 047 public void HoldOut(int testPercent, int repetitions) { 048 049 try { 050 051 // Obtain the time 052 long t = (new Date()).getTime(); 053 int numberOfCycles = 0; 054 055 // Run the precycle to load the case base 056 LogManager.getLogger().info("Running precycle()"); 057 CBRCaseBase caseBase = app.preCycle(); 058 059 if (!(caseBase instanceof CachedLinealCaseBase)) 060 LogManager.getLogger().warn("Evaluation should be executed using a cached case base"); 061 062 List<CBRCase> originalCases = new ArrayList<>(caseBase.getCases()); 063 064 int totalSteps = ((originalCases.size() * testPercent) / 100); 065 totalSteps = totalSteps*repetitions; 066 ProgressController.init(getClass(),"Hold Out Evaluation", totalSteps); 067 068 // For each repetition 069 for (int rep = 0; rep < repetitions; rep++) { 070 071 List<CBRCase> querySet = new ArrayList<>(); 072 List<CBRCase> caseBaseSet = new ArrayList<>(); 073 074 // Split the case base 075 splitCaseBase(originalCases, querySet, caseBaseSet, testPercent); 076 077 // Clear the caseBase 078 caseBase.forgetCases(originalCases); 079 080 // Set the cases that acts as case base in this repetition 081 caseBase.learnCases(caseBaseSet); 082 083 // Run cycle for each case in querySet 084 for (CBRCase c : querySet) { 085 // Run the cycle 086 LogManager.getLogger().info("Running cycle() " + numberOfCycles); 087 app.cycle(c); 088 089 ProgressController.step(getClass()); 090 numberOfCycles++; 091 } 092 } 093 094 ProgressController.finish(getClass()); 095 096 //Revert case base to original state 097 caseBase.forgetCases(originalCases); 098 caseBase.learnCases(originalCases); 099 100 // Run the poscycle to finish the application 101 LogManager.getLogger().info("Running postcycle()"); 102 app.postCycle(); 103 104 t = (new Date()).getTime() - t; 105 106 // Obtain and complete the evaluation result 107 report.setTotalTime(t); 108 report.setNumberOfCycles(numberOfCycles); 109 110 } catch (Exception e) { 111 LogManager.getLogger().error(e); 112 } 113 114 } 115 116 /** 117 * Splits the case base in two sets: queries and case base 118 * @param holeCaseBase Complete original case base 119 * @param querySet Output param where queries are stored 120 * @param casebaseSet Output param where case base is stored 121 * @param testPercent Percentage of cases used as queries 122 */ 123 protected void splitCaseBase(Collection<CBRCase> holeCaseBase, List<CBRCase> querySet, List<CBRCase> casebaseSet, int testPercent) { 124 querySet.clear(); 125 casebaseSet.clear(); 126 127 int querySetSize = (holeCaseBase.size() * testPercent) / 100; 128 casebaseSet.addAll(holeCaseBase); 129 130 for (int i = 0; i < querySetSize; i++) { 131 int random = (int) (Math.random() * casebaseSet.size()); 132 CBRCase _case = casebaseSet.get(random); 133 casebaseSet.remove(random); 134 querySet.add(_case); 135 } 136 } 137 138}