001/** 002 * DetailedHoldOutEvaluator.java 003 * jCOLIBRI2 framework. 004 * @author Juan A. Recio-Garc�a. 005 * @author Lisa Cummins. 006 * GAIA - Group for Artificial Intelligence Applications 007 * http://gaia.fdi.ucm.es 008 * 23/07/2007 009 */ 010package es.ucm.fdi.gaia.jcolibri.extensions.maintenance_evaluation.evaluators; 011 012import org.apache.logging.log4j.LogManager; 013 014import es.ucm.fdi.gaia.jcolibri.casebase.CachedLinealCaseBase; 015import es.ucm.fdi.gaia.jcolibri.cbrcore.CBRCase; 016import es.ucm.fdi.gaia.jcolibri.cbrcore.CBRCaseBase; 017import es.ucm.fdi.gaia.jcolibri.extensions.maintenance_evaluation.MaintenanceEvaluator; 018import es.ucm.fdi.gaia.jcolibri.util.ProgressController; 019 020import java.util.ArrayList; 021import java.util.Date; 022import java.util.List; 023 024/** 025 * This evaluation splits the case base in two sets: a training set and a test set. 026 * The training set is used as a case-base. It is maintained and then the cases in the 027 * test set are used as queries to evaluate the training set. 028 * This process is performed serveral times. 029 * 030 * @author Lisa Cummins. 031 * @author Juan A. Recio Garc�a - GAIA http://gaia.fdi.ucm.es 032 */ 033public class MaintenanceHoldOutEvaluator extends MaintenanceEvaluator 034{ 035 /** 036 * Performs the Hold-Out evaluation. 037 * @param testPercent percentage of the case base used as queries. 038 * The case base is split randomly in each repetition. 039 * @param repetitions number of repetitions. 040 */ 041 public void HoldOut(int testPercent, int repetitions) { 042 try { 043 // Obtain the time 044 long t = (new Date()).getTime(); 045 int numberOfCycles = 0; 046 047 // Run the precycle to load the case base 048 LogManager.getLogger().info("Running precycle()"); 049 CBRCaseBase caseBase = app.preCycle(); 050 051 if (!(caseBase instanceof CachedLinealCaseBase)) 052 LogManager.getLogger().warn("Evaluation should be executed using a cached case base"); 053 054 List<CBRCase> originalCases = new ArrayList<CBRCase>( 055 caseBase.getCases()); 056 057 int totalSteps = ((originalCases.size() * testPercent) / 100); 058 totalSteps = totalSteps*repetitions; 059 ProgressController.init(getClass(),"Hold Out Evaluation", totalSteps); 060 061 // For each repetition 062 for (int rep = 0; rep < repetitions; rep++) 063 { List<CBRCase> querySet = new ArrayList<CBRCase>(); 064 prepareCases(originalCases, querySet, testPercent, caseBase); 065 066 // Run cycle for each case in querySet 067 for (CBRCase c : querySet) 068 { // Run the cycle 069 LogManager.getLogger().info("Running cycle() " + numberOfCycles); 070 071 app.cycle(c); 072 073 ProgressController.step(getClass()); 074 numberOfCycles++; 075 } 076 } 077 078 ProgressController.finish(getClass()); 079 080 //Revert case base to original state 081 caseBase.forgetCases(originalCases); 082 caseBase.learnCases(originalCases); 083 084 // Run the poscycle to finish the application 085 LogManager.getLogger().info("Running postcycle()"); 086 app.postCycle(); 087 088 t = (new Date()).getTime() - t; 089 090 // Obtain and complete the evaluation result 091 report.setTotalTime(t); 092 report.setNumberOfCycles(numberOfCycles); 093 094 } catch (Exception e) { 095 System.out.println(e); 096 } 097 } 098 099 /** 100 * Prepares the cases for evaluation by setting up test and training sets 101 * @param originalCases Complete original set of cases 102 * @param querySet Where queries are to be stored 103 * @param testPercent Percentage of cases used as queries 104 * @param caseBase The case base 105 */ 106 protected void prepareCases(List<CBRCase> originalCases, List<CBRCase> querySet, 107 int testPercent, CBRCaseBase caseBase) 108 { 109 ArrayList<CBRCase> caseBaseSet = new ArrayList<CBRCase>(); 110 111 // Split the case base 112 splitCaseBase(originalCases, querySet, caseBaseSet, testPercent); 113 114 // Clear the caseBase 115 caseBase.forgetCases(originalCases); 116 117 // Set the cases that acts as case base in this repetition 118 caseBase.learnCases(caseBaseSet); 119 120 if(this.simConfig != null && this.editMethod != null) 121 { // Perform maintenance on this case base 122 editCaseBase(caseBase); 123 } 124 } 125 126 /** 127 * Splits the case base in two sets: queries and case base 128 * 129 * @param wholeCaseBase 130 * Complete original case base 131 * @param querySet 132 * Output param where queries are stored 133 * @param casebaseSet 134 * Output param where case base is stored 135 * @param testPercent 136 * Percentage of cases used as queries 137 */ 138 public static void splitCaseBase(List<CBRCase> wholeCaseBase, 139 List<CBRCase> querySet, List<CBRCase> casebaseSet, int testPercent) 140 { querySet.clear(); 141 casebaseSet.clear(); 142 143 int querySetSize = (wholeCaseBase.size() * testPercent) / 100; 144 casebaseSet.addAll(wholeCaseBase); 145 146 for (int i = 0; i < querySetSize; i++) 147 { int random = (int) (Math.random() * casebaseSet.size()); 148 CBRCase _case = casebaseSet.get(random); 149 casebaseSet.remove(random); 150 querySet.add(_case); 151 } 152 } 153}