001/* 002 * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016 017package org.tribuo.common.xgboost; 018 019import com.google.protobuf.Any; 020import com.google.protobuf.ByteString; 021import com.google.protobuf.InvalidProtocolBufferException; 022import com.oracle.labs.mlrg.olcut.util.MutableDouble; 023import com.oracle.labs.mlrg.olcut.util.Pair; 024import ml.dmlc.xgboost4j.java.Booster; 025import ml.dmlc.xgboost4j.java.XGBoost; 026import ml.dmlc.xgboost4j.java.XGBoostError; 027import org.tribuo.Dataset; 028import org.tribuo.Example; 029import org.tribuo.Excuse; 030import org.tribuo.ImmutableFeatureMap; 031import org.tribuo.ImmutableOutputInfo; 032import org.tribuo.Model; 033import org.tribuo.Output; 034import org.tribuo.Prediction; 035import org.tribuo.common.xgboost.XGBoostTrainer.DMatrixTuple; 036import org.tribuo.common.xgboost.protos.XGBoostModelProto; 037import org.tribuo.impl.ModelDataCarrier; 038import org.tribuo.protos.ProtoUtil; 039import org.tribuo.protos.core.ModelProto; 040import org.tribuo.provenance.ModelProvenance; 041import org.tribuo.provenance.TrainerProvenance; 042 043import java.io.IOException; 044import java.io.ObjectInputStream; 045import java.io.ObjectOutputStream; 046import java.lang.reflect.InvocationTargetException; 047import java.util.ArrayList; 048import java.util.Collections; 049import java.util.Comparator; 050import java.util.HashMap; 051import java.util.List; 052import java.util.Map; 053import java.util.Optional; 054import java.util.PriorityQueue; 055import java.util.logging.Level; 056import java.util.logging.Logger; 057import java.util.stream.Collectors; 058 059/** 060 * A {@link Model} which wraps around a XGBoost.Booster. 061 * <p> 062 * XGBoost is a fast implementation of gradient boosted decision trees. 063 * <p> 064 * Throws IllegalStateException if the XGBoost C++ library fails to load or throws an exception. 065 * <p> 066 * See: 067 * <pre> 068 * Chen T, Guestrin C. 069 * "XGBoost: A Scalable Tree Boosting System" 070 * Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, 2016. 071 * </pre> 072 * and for the original algorithm: 073 * <pre> 074 * Friedman JH. 075 * "Greedy Function Approximation: a Gradient Boosting Machine" 076 * Annals of statistics, 2001. 077 * </pre> 078 * <p> 079 * N.B.: XGBoost4J wraps the native C implementation of xgboost that links to various C libraries, including libgomp 080 * and glibc (on Linux). If you're running on Alpine, which does not natively use glibc, you'll need to install glibc 081 * into the container. 082 * On the macOS binary on Maven Central is compiled without 083 * OpenMP support, meaning that XGBoost is single threaded on macOS. You can recompile the macOS binary with 084 * OpenMP support after installing libomp from homebrew if necessary. 085 */ 086public final class XGBoostModel<T extends Output<T>> extends Model<T> { 087 private static final long serialVersionUID = 4L; 088 089 private static final Logger logger = Logger.getLogger(XGBoostModel.class.getName()); 090 091 /** 092 * Protobuf serialization version. 093 */ 094 public static final int CURRENT_VERSION = 0; 095 096 private final XGBoostOutputConverter<T> converter; 097 098 // Used to signal if the model has been rewritten to fix the issue with multidimensional XGBoost regression models in 4.0 and 4.1.0. 099 private boolean regression41MappingFix; 100 101 /** 102 * The XGBoost4J Boosters. 103 */ 104 protected transient List<Booster> models; 105 106 XGBoostModel(String name, ModelProvenance description, 107 ImmutableFeatureMap featureIDMap, ImmutableOutputInfo<T> labelIDMap, 108 List<Booster> models, XGBoostOutputConverter<T> converter) { 109 super(name,description,featureIDMap,labelIDMap,converter.generatesProbabilities()); 110 this.converter = converter; 111 this.models = models; 112 this.regression41MappingFix = true; 113 } 114 115 /** 116 * Deserialization factory. 117 * @param version The serialized object version. 118 * @param className The class name. 119 * @param message The serialized data. 120 * @throws InvalidProtocolBufferException If the protobuf could not be parsed from the {@code message}. 121 * @throws XGBoostError If the XGBoost byte array failed to parse. 122 * @throws IOException If the XGBoost byte array failed to parse. 123 * @return The deserialized object. 124 */ 125 public static XGBoostModel<?> deserializeFromProto(int version, String className, Any message) throws InvalidProtocolBufferException, XGBoostError, IOException { 126 if (version < 0 || version > CURRENT_VERSION) { 127 throw new IllegalArgumentException("Unknown version " + version + ", this class supports at most version " + CURRENT_VERSION); 128 } 129 XGBoostModelProto proto = message.unpack(XGBoostModelProto.class); 130 131 XGBoostOutputConverter<?> converter = ProtoUtil.deserialize(proto.getConverter()); 132 Class<?> converterWitness = converter.getTypeWitness(); 133 ModelDataCarrier<?> carrier = ModelDataCarrier.deserialize(proto.getMetadata()); 134 if (!carrier.outputDomain().getOutput(0).getClass().equals(converterWitness)) { 135 throw new IllegalStateException("Invalid protobuf, output domain does not match the converter, found " + carrier.outputDomain().getClass() + " and " + converterWitness); 136 } 137 List<Booster> models = new ArrayList<>(); 138 for (ByteString b : proto.getModelsList()) { 139 models.add(XGBoost.loadModel(b.toByteArray())); 140 } 141 if (models.isEmpty()) { 142 throw new IllegalStateException("Invalid protobuf, no XGBoost models were found"); 143 } 144 145 @SuppressWarnings({"rawtypes","unchecked"}) // guarded by getClass check on the converter and domain above 146 XGBoostModel<?> model = new XGBoostModel(carrier.name(),carrier.provenance(),carrier.featureDomain(),carrier.outputDomain(),models,converter); 147 return model; 148 } 149 150 /** 151 * Returns an unmodifiable list containing a copy of each model. 152 * <p> 153 * As XGBoost4J models don't expose a copy constructor this requires 154 * serializing each model to a byte array and rebuilding it, and is thus quite expensive. 155 * @return A copy of all of the models. 156 */ 157 public List<Booster> getInnerModels() { 158 List<Booster> copy = new ArrayList<>(); 159 160 for (Booster m : models) { 161 copy.add(copyModel(m)); 162 } 163 164 return Collections.unmodifiableList(copy); 165 } 166 167 /** 168 * Sets the number of threads to use at prediction time. 169 * <p> 170 * If set to 0 sets nthreads = num hardware threads. 171 * @param threads The new number of threads. 172 */ 173 public void setNumThreads(int threads) { 174 if (threads > -1) { 175 try { 176 for (Booster model : models) { 177 model.setParam("nthread", threads); 178 } 179 } catch (XGBoostError e) { 180 logger.log(Level.SEVERE, "XGBoost threw an error", e); 181 throw new IllegalStateException(e); 182 } 183 } 184 } 185 186 /** 187 * Uses the model to predict the labels for multiple examples contained in 188 * a data set. 189 * @param examples the data set containing the examples to predict. 190 * @return the results of the predictions, in the same order as the 191 * data set generates the example. 192 */ 193 @Override 194 public List<Prediction<T>> predict(Dataset<T> examples) { 195 return predict(examples.getData()); 196 } 197 198 /** 199 * Uses the model to predict the label for multiple examples. 200 * @param examples the examples to predict. 201 * @return the results of the prediction, in the same order as the 202 * examples. 203 */ 204 @Override 205 public List<Prediction<T>> predict(Iterable<Example<T>> examples) { 206 try { 207 DMatrixTuple<T> testMatrix = XGBoostTrainer.convertExamples(examples,featureIDMap); 208 List<float[][]> outputs = new ArrayList<>(); 209 for (Booster model : models) { 210 outputs.add(model.predict(testMatrix.data)); 211 } 212 213 int[] numValidFeatures = testMatrix.numValidFeatures; 214 Example<T>[] exampleArray = testMatrix.examples; 215 return converter.convertBatchOutput(outputIDInfo,outputs,numValidFeatures,exampleArray); 216 } catch (XGBoostError e) { 217 logger.log(Level.SEVERE, "XGBoost threw an error", e); 218 throw new IllegalStateException(e); 219 } 220 221 } 222 223 @Override 224 public Prediction<T> predict(Example<T> example) { 225 try { 226 DMatrixTuple<T> testData = XGBoostTrainer.convertExample(example,featureIDMap); 227 List<float[]> outputs = new ArrayList<>(); 228 for (Booster model : models) { 229 outputs.add(model.predict(testData.data)[0]); 230 } 231 Prediction<T> pred = converter.convertOutput(outputIDInfo,outputs,testData.numValidFeatures[0],example); 232 return pred; 233 } catch (XGBoostError e) { 234 logger.log(Level.SEVERE, "XGBoost threw an error", e); 235 throw new IllegalStateException(e); 236 } 237 } 238 239 /** 240 * Creates objects to report feature importance metrics for XGBoost. See the documentation of {@link XGBoostFeatureImportance} 241 * for more information on what those metrics mean. Typically this list will contain a single instance for the entire 242 * model. For multidimensional regression the list will have one entry per dimension, in dimension order. 243 * @return The feature importance object(s). 244 */ 245 public List<XGBoostFeatureImportance> getFeatureImportance() { 246 return models.stream().map(b -> new XGBoostFeatureImportance(b, this)).collect(Collectors.toList()); 247 } 248 249 @Override 250 public Map<String, List<Pair<String,Double>>> getTopFeatures(int n) { 251 try { 252 int maxFeatures = n < 0 ? featureIDMap.size() : n; 253 Map<String, List<Pair<String,Double>>> map = new HashMap<>(); 254 for (int i = 0; i < models.size(); i++) { 255 Booster model = models.get(i); 256 Map<String, MutableDouble> outputMap = new HashMap<>(); 257 Map<String, Integer> xgboostMap = model.getFeatureScore(""); 258 for (Map.Entry<String,Integer> f : xgboostMap.entrySet()) { 259 int id = Integer.parseInt(f.getKey().substring(1)); 260 String name = featureIDMap.get(id).getName(); 261 MutableDouble curVal = outputMap.computeIfAbsent(name,(k)->new MutableDouble()); 262 curVal.increment(f.getValue()); 263 } 264 265 Comparator<Pair<String, Double>> comparator = Comparator.comparingDouble(p -> Math.abs(p.getB())); 266 PriorityQueue<Pair<String,Double>> q = new PriorityQueue<>(maxFeatures,comparator); 267 for (Map.Entry<String,MutableDouble> e : outputMap.entrySet()) { 268 Pair<String,Double> cur = new Pair<>(e.getKey(), e.getValue().doubleValue()); 269 270 if (q.size() < maxFeatures) { 271 q.offer(cur); 272 } else if (comparator.compare(cur,q.peek()) > 0) { 273 q.poll(); 274 q.offer(cur); 275 } 276 } 277 List<Pair<String,Double>> list = new ArrayList<>(); 278 while(q.size() > 0) { 279 list.add(q.poll()); 280 } 281 Collections.reverse(list); 282 283 if (models.size() == 1) { 284 map.put(Model.ALL_OUTPUTS, list); 285 } else { 286 String dimensionName = outputIDInfo.getOutput(i).toString(); 287 map.put(dimensionName, list); 288 } 289 } 290 291 return map; 292 } catch (XGBoostError e) { 293 logger.log(Level.SEVERE, "XGBoost threw an error", e); 294 return Collections.emptyMap(); 295 } 296 } 297 298 /** 299 * Returns the string model dumps from each Booster. 300 * @return The model dumps. 301 */ 302 public List<String[]> getModelDump() { 303 try { 304 List<String[]> list = new ArrayList<>(); 305 for (Booster m : models) { 306 list.add(m.getModelDump("", true)); 307 } 308 return list; 309 } catch (XGBoostError e) { 310 throw new IllegalStateException(e); 311 } 312 } 313 314 @Override 315 public Optional<Excuse<T>> getExcuse(Example<T> example) { 316 return Optional.empty(); 317 } 318 319 /** 320 * Copies a single XGBoost Booster by serializing and deserializing it. 321 * @param booster The booster to copy. 322 * @return A deep copy of the booster. 323 */ 324 static Booster copyModel(Booster booster) { 325 try { 326 byte[] serialisedBooster = booster.toByteArray(); 327 return XGBoost.loadModel(serialisedBooster); 328 } catch (XGBoostError | IOException e) { 329 throw new IllegalStateException("Unable to copy XGBoost model.",e); 330 } 331 } 332 333 @Override 334 protected Model<T> copy(String newName, ModelProvenance newProvenance) { 335 List<Booster> newModels = new ArrayList<>(); 336 for (Booster model : models) { 337 newModels.add(copyModel(model)); 338 } 339 return new XGBoostModel<>(newName, newProvenance, featureIDMap, outputIDInfo, newModels, converter); 340 } 341 342 @Override 343 public ModelProto serialize() { 344 ModelDataCarrier<T> carrier = createDataCarrier(); 345 346 XGBoostModelProto.Builder modelBuilder = XGBoostModelProto.newBuilder(); 347 modelBuilder.setMetadata(carrier.serialize()); 348 modelBuilder.setConverter(converter.serialize()); 349 try { 350 for (Booster b : models) { 351 modelBuilder.addModels(ByteString.copyFrom(b.toByteArray())); 352 } 353 } catch (XGBoostError e) { 354 throw new IllegalStateException("Failed to serialize XGBoost model"); 355 } 356 357 ModelProto.Builder builder = ModelProto.newBuilder(); 358 builder.setSerializedData(Any.pack(modelBuilder.build())); 359 builder.setClassName(XGBoostModel.class.getName()); 360 builder.setVersion(CURRENT_VERSION); 361 362 return builder.build(); 363 } 364 365 private void writeObject(ObjectOutputStream out) throws IOException { 366 out.defaultWriteObject(); 367 try { 368 out.writeInt(models.size()); 369 for (Booster model : models) { 370 byte[] serialisedBooster = model.toByteArray(); 371 out.writeObject(serialisedBooster); 372 } 373 } catch (XGBoostError e) { 374 throw new IOException("Failed to serialize the XGBoost model",e); 375 } 376 } 377 378 private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { 379 in.defaultReadObject(); 380 try { 381 this.models = new ArrayList<>(); 382 int numModels = in.readInt(); 383 for (int i = 0; i < numModels; i++) { 384 // Now read in the byte array and rebuild each Booster 385 byte[] serialisedBooster = (byte[]) in.readObject(); 386 this.models.add(XGBoost.loadModel(serialisedBooster)); 387 } 388 try { 389 Class<?> regressionClass = Class.forName("org.tribuo.regression.ImmutableRegressionInfo"); 390 String tribuoVersion = (String) provenance.getTrainerProvenance().getInstanceValues().get(TrainerProvenance.TRIBUO_VERSION_STRING).getValue(); 391 if (regressionClass.isInstance(outputIDInfo) && !regression41MappingFix && 392 (tribuoVersion.startsWith("4.0.0") || tribuoVersion.startsWith("4.0.1") || tribuoVersion.startsWith("4.0.2") || tribuoVersion.startsWith("4.1.0") 393 // This is explicit to catch the test model which has a 4.1.1-SNAPSHOT Tribuo version. 394 || tribuoVersion.equals("4.1.1-SNAPSHOT"))) { 395 // rewrite the model stream 396 regression41MappingFix = true; 397 int[] mapping = (int[]) regressionClass.getDeclaredMethod("getIDtoNaturalOrderMapping").invoke(outputIDInfo); 398 List<Booster> copy = new ArrayList<>(models); 399 for (int i = 0; i < mapping.length; i++) { 400 copy.set(i,models.get(mapping[i])); 401 } 402 this.models = copy; 403 } 404 } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { 405 throw new RuntimeException("Failed to rewrite 4.1.0 or earlier regression model due to a reflection failure.",e); 406 } catch (ClassNotFoundException e) { 407 // pass as this isn't a regression model as otherwise it would have thrown ClassNotFoundException 408 // during the reading of the input stream. 409 } 410 } catch (XGBoostError e) { 411 throw new IOException("Failed to deserialize the XGBoost model",e); 412 } 413 } 414}