001/*
002 * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 *     http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016
017package org.tribuo.common.xgboost;
018
019import com.google.protobuf.Any;
020import com.google.protobuf.ByteString;
021import com.google.protobuf.InvalidProtocolBufferException;
022import com.oracle.labs.mlrg.olcut.util.MutableDouble;
023import com.oracle.labs.mlrg.olcut.util.Pair;
024import ml.dmlc.xgboost4j.java.Booster;
025import ml.dmlc.xgboost4j.java.XGBoost;
026import ml.dmlc.xgboost4j.java.XGBoostError;
027import org.tribuo.Dataset;
028import org.tribuo.Example;
029import org.tribuo.Excuse;
030import org.tribuo.ImmutableFeatureMap;
031import org.tribuo.ImmutableOutputInfo;
032import org.tribuo.Model;
033import org.tribuo.Output;
034import org.tribuo.Prediction;
035import org.tribuo.common.xgboost.XGBoostTrainer.DMatrixTuple;
036import org.tribuo.common.xgboost.protos.XGBoostModelProto;
037import org.tribuo.impl.ModelDataCarrier;
038import org.tribuo.protos.ProtoUtil;
039import org.tribuo.protos.core.ModelProto;
040import org.tribuo.provenance.ModelProvenance;
041import org.tribuo.provenance.TrainerProvenance;
042
043import java.io.IOException;
044import java.io.ObjectInputStream;
045import java.io.ObjectOutputStream;
046import java.lang.reflect.InvocationTargetException;
047import java.util.ArrayList;
048import java.util.Collections;
049import java.util.Comparator;
050import java.util.HashMap;
051import java.util.List;
052import java.util.Map;
053import java.util.Optional;
054import java.util.PriorityQueue;
055import java.util.logging.Level;
056import java.util.logging.Logger;
057import java.util.stream.Collectors;
058
059/**
060 * A {@link Model} which wraps around a XGBoost.Booster.
061 * <p>
062 * XGBoost is a fast implementation of gradient boosted decision trees.
063 * <p>
064 * Throws IllegalStateException if the XGBoost C++ library fails to load or throws an exception.
065 * <p>
066 * See:
067 * <pre>
068 * Chen T, Guestrin C.
069 * "XGBoost: A Scalable Tree Boosting System"
070 * Proceedings of the 22nd ACM SIGKDD International Conference on Knowledge Discovery and Data Mining, 2016.
071 * </pre>
072 * and for the original algorithm:
073 * <pre>
074 * Friedman JH.
075 * "Greedy Function Approximation: a Gradient Boosting Machine"
076 * Annals of statistics, 2001.
077 * </pre>
078 * <p>
079 * N.B.: XGBoost4J wraps the native C implementation of xgboost that links to various C libraries, including libgomp
080 * and glibc (on Linux). If you're running on Alpine, which does not natively use glibc, you'll need to install glibc
081 * into the container.
082 * On the macOS binary on Maven Central is compiled without
083 * OpenMP support, meaning that XGBoost is single threaded on macOS. You can recompile the macOS binary with
084 * OpenMP support after installing libomp from homebrew if necessary.
085 */
086public final class XGBoostModel<T extends Output<T>> extends Model<T> {
087    private static final long serialVersionUID = 4L;
088
089    private static final Logger logger = Logger.getLogger(XGBoostModel.class.getName());
090
091    /**
092     * Protobuf serialization version.
093     */
094    public static final int CURRENT_VERSION = 0;
095
096    private final XGBoostOutputConverter<T> converter;
097
098    // Used to signal if the model has been rewritten to fix the issue with multidimensional XGBoost regression models in 4.0 and 4.1.0.
099    private boolean regression41MappingFix;
100
101    /**
102     * The XGBoost4J Boosters.
103     */
104    protected transient List<Booster> models;
105
106    XGBoostModel(String name, ModelProvenance description,
107                 ImmutableFeatureMap featureIDMap, ImmutableOutputInfo<T> labelIDMap,
108                 List<Booster> models, XGBoostOutputConverter<T> converter) {
109        super(name,description,featureIDMap,labelIDMap,converter.generatesProbabilities());
110        this.converter = converter;
111        this.models = models;
112        this.regression41MappingFix = true;
113    }
114
115    /**
116     * Deserialization factory.
117     * @param version The serialized object version.
118     * @param className The class name.
119     * @param message The serialized data.
120     * @throws InvalidProtocolBufferException If the protobuf could not be parsed from the {@code message}.
121     * @throws XGBoostError If the XGBoost byte array failed to parse.
122     * @throws IOException If the XGBoost byte array failed to parse.
123     * @return The deserialized object.
124     */
125    public static XGBoostModel<?> deserializeFromProto(int version, String className, Any message) throws InvalidProtocolBufferException, XGBoostError, IOException {
126        if (version < 0 || version > CURRENT_VERSION) {
127            throw new IllegalArgumentException("Unknown version " + version + ", this class supports at most version " + CURRENT_VERSION);
128        }
129        XGBoostModelProto proto = message.unpack(XGBoostModelProto.class);
130
131        XGBoostOutputConverter<?> converter = ProtoUtil.deserialize(proto.getConverter());
132        Class<?> converterWitness = converter.getTypeWitness();
133        ModelDataCarrier<?> carrier = ModelDataCarrier.deserialize(proto.getMetadata());
134        if (!carrier.outputDomain().getOutput(0).getClass().equals(converterWitness)) {
135            throw new IllegalStateException("Invalid protobuf, output domain does not match the converter, found " + carrier.outputDomain().getClass() + " and " + converterWitness);
136        }
137        List<Booster> models = new ArrayList<>();
138        for (ByteString b : proto.getModelsList()) {
139            models.add(XGBoost.loadModel(b.toByteArray()));
140        }
141        if (models.isEmpty()) {
142            throw new IllegalStateException("Invalid protobuf, no XGBoost models were found");
143        }
144
145        @SuppressWarnings({"rawtypes","unchecked"}) // guarded by getClass check on the converter and domain above
146        XGBoostModel<?> model = new XGBoostModel(carrier.name(),carrier.provenance(),carrier.featureDomain(),carrier.outputDomain(),models,converter);
147        return model;
148    }
149
150    /**
151     * Returns an unmodifiable list containing a copy of each model.
152     * <p>
153     * As XGBoost4J models don't expose a copy constructor this requires
154     * serializing each model to a byte array and rebuilding it, and is thus quite expensive.
155     * @return A copy of all of the models.
156     */
157    public List<Booster> getInnerModels() {
158        List<Booster> copy = new ArrayList<>();
159
160        for (Booster m : models) {
161            copy.add(copyModel(m));
162        }
163
164        return Collections.unmodifiableList(copy);
165    }
166
167    /**
168     * Sets the number of threads to use at prediction time.
169     * <p>
170     * If set to 0 sets nthreads = num hardware threads.
171     * @param threads The new number of threads.
172     */
173    public void setNumThreads(int threads) {
174        if (threads > -1) {
175            try {
176                for (Booster model : models) {
177                    model.setParam("nthread", threads);
178                }
179            } catch (XGBoostError e) {
180                logger.log(Level.SEVERE, "XGBoost threw an error", e);
181                throw new IllegalStateException(e);
182            }
183        }
184    }
185
186    /**
187     * Uses the model to predict the labels for multiple examples contained in
188     * a data set.
189     * @param examples the data set containing the examples to predict.
190     * @return the results of the predictions, in the same order as the
191     * data set generates the example.
192     */
193    @Override
194    public List<Prediction<T>> predict(Dataset<T> examples) {
195        return predict(examples.getData());
196    }
197
198    /**
199     * Uses the model to predict the label for multiple examples.
200     * @param examples the examples to predict.
201     * @return the results of the prediction, in the same order as the
202     * examples.
203     */
204    @Override
205    public List<Prediction<T>> predict(Iterable<Example<T>> examples) {
206        try {
207            DMatrixTuple<T> testMatrix = XGBoostTrainer.convertExamples(examples,featureIDMap);
208            List<float[][]> outputs = new ArrayList<>();
209            for (Booster model : models) {
210                outputs.add(model.predict(testMatrix.data));
211            }
212
213            int[] numValidFeatures = testMatrix.numValidFeatures;
214            Example<T>[] exampleArray = testMatrix.examples;
215            return converter.convertBatchOutput(outputIDInfo,outputs,numValidFeatures,exampleArray);
216        } catch (XGBoostError e) {
217            logger.log(Level.SEVERE, "XGBoost threw an error", e);
218            throw new IllegalStateException(e);
219        }
220
221    }
222
223    @Override
224    public Prediction<T> predict(Example<T> example) {
225        try {
226            DMatrixTuple<T> testData = XGBoostTrainer.convertExample(example,featureIDMap);
227            List<float[]> outputs = new ArrayList<>();
228            for (Booster model : models) {
229                outputs.add(model.predict(testData.data)[0]);
230            }
231            Prediction<T> pred = converter.convertOutput(outputIDInfo,outputs,testData.numValidFeatures[0],example);
232            return pred;
233        } catch (XGBoostError e) {
234            logger.log(Level.SEVERE, "XGBoost threw an error", e);
235            throw new IllegalStateException(e);
236        }
237    }
238
239    /**
240     * Creates objects to report feature importance metrics for XGBoost. See the documentation of {@link XGBoostFeatureImportance}
241     * for more information on what those metrics mean. Typically this list will contain a single instance for the entire
242     * model. For multidimensional regression the list will have one entry per dimension, in dimension order.
243     * @return The feature importance object(s).
244     */
245    public List<XGBoostFeatureImportance> getFeatureImportance() {
246        return models.stream().map(b -> new XGBoostFeatureImportance(b, this)).collect(Collectors.toList());
247    }
248
249    @Override
250    public Map<String, List<Pair<String,Double>>> getTopFeatures(int n) {
251        try {
252            int maxFeatures = n < 0 ? featureIDMap.size() : n;
253            Map<String, List<Pair<String,Double>>> map = new HashMap<>();
254            for (int i = 0; i < models.size(); i++) {
255                Booster model = models.get(i);
256                Map<String, MutableDouble> outputMap = new HashMap<>();
257                Map<String, Integer> xgboostMap = model.getFeatureScore("");
258                for (Map.Entry<String,Integer> f : xgboostMap.entrySet()) {
259                    int id = Integer.parseInt(f.getKey().substring(1));
260                    String name = featureIDMap.get(id).getName();
261                    MutableDouble curVal = outputMap.computeIfAbsent(name,(k)->new MutableDouble());
262                    curVal.increment(f.getValue());
263                }
264
265                Comparator<Pair<String, Double>> comparator = Comparator.comparingDouble(p -> Math.abs(p.getB()));
266                PriorityQueue<Pair<String,Double>> q = new PriorityQueue<>(maxFeatures,comparator);
267                for (Map.Entry<String,MutableDouble> e : outputMap.entrySet()) {
268                    Pair<String,Double> cur = new Pair<>(e.getKey(), e.getValue().doubleValue());
269
270                    if (q.size() < maxFeatures) {
271                        q.offer(cur);
272                    } else if (comparator.compare(cur,q.peek()) > 0) {
273                        q.poll();
274                        q.offer(cur);
275                    }
276                }
277                List<Pair<String,Double>> list = new ArrayList<>();
278                while(q.size() > 0) {
279                    list.add(q.poll());
280                }
281                Collections.reverse(list);
282
283                if (models.size() == 1) {
284                    map.put(Model.ALL_OUTPUTS, list);
285                } else {
286                    String dimensionName = outputIDInfo.getOutput(i).toString();
287                    map.put(dimensionName, list);
288                }
289            }
290
291            return map;
292        } catch (XGBoostError e) {
293            logger.log(Level.SEVERE, "XGBoost threw an error", e);
294            return Collections.emptyMap();
295        }
296    }
297
298    /**
299     * Returns the string model dumps from each Booster.
300     * @return The model dumps.
301     */
302    public List<String[]> getModelDump() {
303        try {
304            List<String[]> list = new ArrayList<>();
305            for (Booster m : models) {
306                list.add(m.getModelDump("", true));
307            }
308            return list;
309        } catch (XGBoostError e) {
310            throw new IllegalStateException(e);
311        }
312    }
313
314    @Override
315    public Optional<Excuse<T>> getExcuse(Example<T> example) {
316        return Optional.empty();
317    }
318
319    /**
320     * Copies a single XGBoost Booster by serializing and deserializing it.
321     * @param booster The booster to copy.
322     * @return A deep copy of the booster.
323     */
324    static Booster copyModel(Booster booster) {
325        try {
326            byte[] serialisedBooster = booster.toByteArray();
327            return XGBoost.loadModel(serialisedBooster);
328        } catch (XGBoostError | IOException e) {
329            throw new IllegalStateException("Unable to copy XGBoost model.",e);
330        }
331    }
332
333    @Override
334    protected Model<T> copy(String newName, ModelProvenance newProvenance) {
335        List<Booster> newModels = new ArrayList<>();
336        for (Booster model : models) {
337            newModels.add(copyModel(model));
338        }
339        return new XGBoostModel<>(newName, newProvenance, featureIDMap, outputIDInfo, newModels, converter);
340    }
341
342    @Override
343    public ModelProto serialize() {
344        ModelDataCarrier<T> carrier = createDataCarrier();
345
346        XGBoostModelProto.Builder modelBuilder = XGBoostModelProto.newBuilder();
347        modelBuilder.setMetadata(carrier.serialize());
348        modelBuilder.setConverter(converter.serialize());
349        try {
350            for (Booster b : models) {
351                modelBuilder.addModels(ByteString.copyFrom(b.toByteArray()));
352            }
353        } catch (XGBoostError e) {
354            throw new IllegalStateException("Failed to serialize XGBoost model");
355        }
356
357        ModelProto.Builder builder = ModelProto.newBuilder();
358        builder.setSerializedData(Any.pack(modelBuilder.build()));
359        builder.setClassName(XGBoostModel.class.getName());
360        builder.setVersion(CURRENT_VERSION);
361
362        return builder.build();
363    }
364
365    private void writeObject(ObjectOutputStream out) throws IOException {
366        out.defaultWriteObject();
367        try {
368            out.writeInt(models.size());
369            for (Booster model : models) {
370                byte[] serialisedBooster = model.toByteArray();
371                out.writeObject(serialisedBooster);
372            }
373        } catch (XGBoostError e) {
374            throw new IOException("Failed to serialize the XGBoost model",e);
375        }
376    }
377
378    private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException {
379        in.defaultReadObject();
380        try {
381            this.models = new ArrayList<>();
382            int numModels = in.readInt();
383            for (int i = 0; i < numModels; i++) {
384                // Now read in the byte array and rebuild each Booster
385                byte[] serialisedBooster = (byte[]) in.readObject();
386                this.models.add(XGBoost.loadModel(serialisedBooster));
387            }
388            try {
389                Class<?> regressionClass = Class.forName("org.tribuo.regression.ImmutableRegressionInfo");
390                String tribuoVersion = (String) provenance.getTrainerProvenance().getInstanceValues().get(TrainerProvenance.TRIBUO_VERSION_STRING).getValue();
391                if (regressionClass.isInstance(outputIDInfo) && !regression41MappingFix &&
392                        (tribuoVersion.startsWith("4.0.0") || tribuoVersion.startsWith("4.0.1") || tribuoVersion.startsWith("4.0.2") || tribuoVersion.startsWith("4.1.0")
393                                // This is explicit to catch the test model which has a 4.1.1-SNAPSHOT Tribuo version.
394                                || tribuoVersion.equals("4.1.1-SNAPSHOT"))) {
395                    // rewrite the model stream
396                    regression41MappingFix = true;
397                    int[] mapping = (int[]) regressionClass.getDeclaredMethod("getIDtoNaturalOrderMapping").invoke(outputIDInfo);
398                    List<Booster> copy = new ArrayList<>(models);
399                    for (int i = 0; i < mapping.length; i++) {
400                        copy.set(i,models.get(mapping[i]));
401                    }
402                    this.models = copy;
403                }
404            } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) {
405                throw new RuntimeException("Failed to rewrite 4.1.0 or earlier regression model due to a reflection failure.",e);
406            } catch (ClassNotFoundException e) {
407                // pass as this isn't a regression model as otherwise it would have thrown ClassNotFoundException
408                // during the reading of the input stream.
409            }
410        } catch (XGBoostError e) {
411            throw new IOException("Failed to deserialize the XGBoost model",e);
412        }
413    }
414}