001/*
002 * Copyright (c) 2021, Oracle and/or its affiliates.
003 *
004 * Licensed under the 2-clause BSD license.
005 *
006 * Redistribution and use in source and binary forms, with or without
007 * modification, are permitted provided that the following conditions are met:
008 *
009 * 1. Redistributions of source code must retain the above copyright notice,
010 *    this list of conditions and the following disclaimer.
011 *
012 * 2. Redistributions in binary form must reproduce the above copyright notice,
013 *    this list of conditions and the following disclaimer in the documentation
014 *    and/or other materials provided with the distribution.
015 *
016 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
017 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
018 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
019 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
020 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
021 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
022 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
023 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
024 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
025 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
026 * POSSIBILITY OF SUCH DAMAGE.
027 */
028
029package com.oracle.labs.mlrg.olcut.config.protobuf;
030
031import com.google.protobuf.InvalidProtocolBufferException;
032import com.google.protobuf.Message;
033import com.google.protobuf.TextFormat;
034import com.oracle.labs.mlrg.olcut.config.protobuf.protos.ListProvenanceProto;
035import com.oracle.labs.mlrg.olcut.config.protobuf.protos.MapProvenanceProto;
036import com.oracle.labs.mlrg.olcut.config.protobuf.protos.ObjectProvenanceProto;
037import com.oracle.labs.mlrg.olcut.config.protobuf.protos.RootProvenanceProto;
038import com.oracle.labs.mlrg.olcut.config.protobuf.protos.SimpleProvenanceProto;
039import com.oracle.labs.mlrg.olcut.provenance.io.FlatMarshalledProvenance;
040import com.oracle.labs.mlrg.olcut.provenance.io.ListMarshalledProvenance;
041import com.oracle.labs.mlrg.olcut.provenance.io.MapMarshalledProvenance;
042import com.oracle.labs.mlrg.olcut.provenance.io.ObjectMarshalledProvenance;
043import com.oracle.labs.mlrg.olcut.provenance.io.ProvenanceSerialization;
044import com.oracle.labs.mlrg.olcut.provenance.io.SimpleMarshalledProvenance;
045import com.oracle.labs.mlrg.olcut.util.MutableLong;
046import com.oracle.labs.mlrg.olcut.util.Pair;
047
048import java.io.BufferedOutputStream;
049import java.io.IOException;
050import java.io.InputStream;
051import java.io.InputStreamReader;
052import java.io.PrintWriter;
053import java.nio.charset.StandardCharsets;
054import java.nio.file.Files;
055import java.nio.file.Path;
056import java.util.ArrayList;
057import java.util.Base64;
058import java.util.HashMap;
059import java.util.List;
060import java.util.Map;
061
062/**
063 * Class for serializing and deserializing provenances to/from protobufs.
064 * <p>
065 * Uses {@link Base64} to encode and decode binary protobufs into and out of Strings if requested, though it
066 * is preferable to use protobuf's built-in text format.
067 */
068public final class ProtoProvenanceSerialization implements ProvenanceSerialization {
069
070    private static final Base64.Encoder base64Encoder = Base64.getEncoder();
071    private static final Base64.Decoder base64Decoder = Base64.getDecoder();
072
073    private final boolean textFormat;
074
075    /**
076     * Construct a ProtoProvenanceSerialization.
077     *
078     * @param textFormat Output a pbtxt.
079     */
080    public ProtoProvenanceSerialization(boolean textFormat) {
081        this.textFormat = textFormat;
082    }
083
084    @Override
085    public String getFileExtension() {
086        return textFormat ? "pbtxt" : "pb";
087    }
088
089    @Override
090    public List<ObjectMarshalledProvenance> deserializeFromFile(Path path) throws IOException {
091        try {
092            InputStream is = Files.newInputStream(path);
093            RootProvenanceProto proto;
094            if (textFormat) {
095                RootProvenanceProto.Builder protoBuilder = RootProvenanceProto.newBuilder();
096                TextFormat.getParser().merge(new InputStreamReader(is,StandardCharsets.UTF_8), protoBuilder);
097                proto = protoBuilder.build();
098            } else {
099                proto = RootProvenanceProto.parseFrom(is);
100            }
101            return deserializeFromProto(proto);
102        } catch (InvalidProtocolBufferException | TextFormat.ParseException e) {
103            throw new IllegalArgumentException("Failed to parse protobuf",e);
104        }
105    }
106
107    @Override
108    public List<ObjectMarshalledProvenance> deserializeFromString(String input) {
109        try {
110            RootProvenanceProto proto;
111            if (textFormat) {
112                RootProvenanceProto.Builder protoBuilder = RootProvenanceProto.newBuilder();
113                TextFormat.getParser().merge(input, protoBuilder);
114                proto = protoBuilder.build();
115            } else {
116                byte[] bytes = base64Decoder.decode(input);
117                proto = RootProvenanceProto.parseFrom(bytes);
118            }
119            return deserializeFromProto(proto);
120        } catch (InvalidProtocolBufferException | TextFormat.ParseException e) {
121            throw new IllegalArgumentException("Failed to parse protobuf",e);
122        }
123    }
124
125    /**
126     * Deserialize the marshalled provenances from the supplied protobuf.
127     * @param proto The protobuf to deserialize.
128     * @return The marshalled provenances encoded in the protobuf.
129     */
130    public List<ObjectMarshalledProvenance> deserializeFromProto(RootProvenanceProto proto) {
131        int totalProtos = proto.getLmpCount() + proto.getMmpCount() + proto.getOmpCount() + proto.getSmpCount();
132        Message[] messages = new Message[totalProtos];
133        // Unpack into map
134        for (ObjectProvenanceProto p : proto.getOmpList()) {
135            int curIndex = p.getIndex();
136            if (messages[curIndex] != null) {
137                throw new IllegalArgumentException("Invalid protobuf found, index " + curIndex + " collided, found '" + p.toString() + " and " + messages[curIndex].toString());
138            } else {
139                messages[curIndex] = p;
140            }
141        }
142        for (SimpleProvenanceProto p : proto.getSmpList()) {
143            int curIndex = p.getIndex();
144            if (messages[curIndex] != null) {
145                throw new IllegalArgumentException("Invalid protobuf found, index " + curIndex + " collided, found '" + p.toString() + " and " + messages[curIndex].toString());
146            } else {
147                messages[curIndex] = p;
148            }
149        }
150        for (MapProvenanceProto p : proto.getMmpList()) {
151            int curIndex = p.getIndex();
152            if (messages[curIndex] != null) {
153                throw new IllegalArgumentException("Invalid protobuf found, index " + curIndex + " collided, found '" + p.toString() + " and " + messages[curIndex].toString());
154            } else {
155                messages[curIndex] = p;
156            }
157        }
158        for (ListProvenanceProto p : proto.getLmpList()) {
159            int curIndex = p.getIndex();
160            if (messages[curIndex] != null) {
161                throw new IllegalArgumentException("Invalid protobuf found, index " + curIndex + " collided, found '" + p.toString() + " and " + messages[curIndex].toString());
162            } else {
163                messages[curIndex] = p;
164            }
165        }
166
167        List<ObjectMarshalledProvenance> outputList = new ArrayList<>();
168
169        // Deserialize the ObjectMarshalledProvenances
170        for (ObjectProvenanceProto p : proto.getOmpList()) {
171            Map<String,FlatMarshalledProvenance> provMap = new HashMap<>();
172            for (Map.Entry<String,Integer> e : p.getValuesMap().entrySet()) {
173                FlatMarshalledProvenance fmp = dispatchMessage(messages,e.getValue());
174                provMap.put(e.getKey(),fmp);
175            }
176
177            ObjectMarshalledProvenance omp = new ObjectMarshalledProvenance(p.getObjectName(),provMap,p.getObjectClassName(),p.getProvenanceClassName());
178            outputList.add(omp);
179        }
180
181        return outputList;
182    }
183
184    /**
185     * Decode the FlatMarshalledProvenance specified by this message index.
186     * @param messages The messages.
187     * @param index The current index.
188     * @return The FlatMarshalledProvenance specified by the index.
189     */
190    private static FlatMarshalledProvenance dispatchMessage(Message[] messages, int index) {
191        Message curMessage = messages[index];
192        if (curMessage instanceof SimpleProvenanceProto) {
193            return decodeSMP((SimpleProvenanceProto) curMessage);
194        } else if (curMessage instanceof ListProvenanceProto) {
195            return decodeLMP(messages, (ListProvenanceProto) curMessage);
196        } else if (curMessage instanceof MapProvenanceProto) {
197            return decodeMMP(messages, (MapProvenanceProto) curMessage);
198        } else {
199            throw new IllegalStateException("Invalid protobuf, a message index points to an ObjectMarshalledProvenance");
200        }
201    }
202
203    /**
204     * Decodes a SimpleProvenanceProto into a SimpleMarshalledProvenance.
205     * @param proto The proto to decode.
206     * @return The decoded provenance.
207     */
208    private static SimpleMarshalledProvenance decodeSMP(SimpleProvenanceProto proto) {
209        return new SimpleMarshalledProvenance(proto.getKey(),proto.getValue(),proto.getProvenanceClassName(),proto.getIsReference(),proto.getAdditional());
210    }
211
212    /**
213     * Decodes a ListProvenanceProto into a ListMarshalledProvenance.
214     * @param messages The messages.
215     * @param proto The proto to decode.
216     * @return The decoded list provenance.
217     */
218    private static ListMarshalledProvenance decodeLMP(Message[] messages, ListProvenanceProto proto) {
219        List<FlatMarshalledProvenance> list = new ArrayList<>();
220
221        for (Integer i : proto.getValuesList()) {
222            list.add(dispatchMessage(messages,i));
223        }
224
225        return new ListMarshalledProvenance(list);
226    }
227
228    /**
229     * Decodes a MapProvenanceProto into a MapMarshalledProvenance.
230     * @param messages The messages.
231     * @param proto The proto to decode.
232     * @return The decoded map provenance.
233     */
234    private static MapMarshalledProvenance decodeMMP(Message[] messages, MapProvenanceProto proto) {
235        Map<String,FlatMarshalledProvenance> map = new HashMap<>();
236
237        for (Map.Entry<String,Integer> e : proto.getValuesMap().entrySet()) {
238            map.put(e.getKey(), dispatchMessage(messages,e.getValue()));
239        }
240
241        return new MapMarshalledProvenance(map);
242    }
243
244    /**
245     * Serialize the marshalled provenances into a protobuf.
246     * @param marshalledProvenances The provenances to serialize.
247     * @return A protobuf encoding all the provenances.
248     */
249    public RootProvenanceProto serializeToProto(List<ObjectMarshalledProvenance> marshalledProvenances) {
250        RootProvenanceProto.Builder builder = RootProvenanceProto.newBuilder();
251
252        // linearizing the provenances
253        MutableLong counter = new MutableLong(0);
254        for (ObjectMarshalledProvenance omp : marshalledProvenances) {
255            convertProvenance(builder, counter, omp);
256        }
257
258        return builder.build();
259    }
260
261    /**
262     * Encodes the ObjectMarshalledProvenance into the RootProvenanceProto.
263     * @param builder The proto builder.
264     * @param counter The index counter.
265     * @param omp The provenance to encode.
266     */
267    private static void convertProvenance(RootProvenanceProto.Builder builder, MutableLong counter, ObjectMarshalledProvenance omp) {
268        ObjectProvenanceProto.Builder ompBuilder = ObjectProvenanceProto.newBuilder();
269
270        ompBuilder.setIndex(counter.intValue());
271        counter.increment();
272
273        ompBuilder.setObjectName(omp.getName());
274        ompBuilder.setObjectClassName(omp.getObjectClassName());
275        ompBuilder.setProvenanceClassName(omp.getProvenanceClassName());
276
277        for (Map.Entry<String, FlatMarshalledProvenance> e : omp.getMap().entrySet()) {
278            int count = dispatchFMP(builder, counter, e.getValue());
279            ompBuilder.putValues(e.getKey(),count);
280        }
281
282        builder.addOmp(ompBuilder.build());
283    }
284
285    /**
286     * Dispatch on the type of FlatMarshalledProvenance.
287     * @param builder The root proto builder.
288     * @param counter The provenance counter.
289     * @param fmp The FMP to dispatch on.
290     */
291    private static int dispatchFMP(RootProvenanceProto.Builder builder, MutableLong counter, FlatMarshalledProvenance fmp) {
292        if (fmp instanceof SimpleMarshalledProvenance) {
293            SimpleMarshalledProvenance smp = (SimpleMarshalledProvenance) fmp;
294            return encodeSMP(builder,counter,smp);
295        } else if (fmp instanceof ListMarshalledProvenance) {
296            ListMarshalledProvenance lmp = (ListMarshalledProvenance) fmp;
297            return encodeLMP(builder,counter,lmp);
298        } else if (fmp instanceof MapMarshalledProvenance) {
299            MapMarshalledProvenance mmp = (MapMarshalledProvenance) fmp;
300            return encodeMMP(builder,counter,mmp);
301        } else {
302            throw new RuntimeException("Should not reach here, unexpected FlatMarshalledProvenance subclass " + fmp.getClass());
303        }
304    }
305
306    /**
307     * Encodes the SimpleMarshalledProvenance into the RootProvenanceProto.
308     * @param builder The proto builder.
309     * @param counter The index counter.
310     * @param smp The provenance to encode.
311     * @return The index of this SimpleMarshalledProvenance
312     */
313    private static int encodeSMP(RootProvenanceProto.Builder builder, MutableLong counter, SimpleMarshalledProvenance smp) {
314        SimpleProvenanceProto.Builder smpBuilder = SimpleProvenanceProto.newBuilder();
315
316        int curIndex = counter.intValue();
317        smpBuilder.setIndex(curIndex);
318        counter.increment();
319
320        smpBuilder.setKey(smp.getKey());
321        smpBuilder.setValue(smp.getValue());
322        smpBuilder.setAdditional(smp.getAdditional());
323        smpBuilder.setProvenanceClassName(smp.getProvenanceClassName());
324        smpBuilder.setIsReference(smp.isReference());
325
326        builder.addSmp(smpBuilder.build());
327        return curIndex;
328    }
329
330    /**
331     * Encodes the ListMarshalledProvenance into the RootProvenanceProto.
332     * @param builder The proto builder.
333     * @param counter The index counter.
334     * @param lmp The provenance to encode.
335     * @return The index of this ListMarshalledProvenance
336     */
337    private static int encodeLMP(RootProvenanceProto.Builder builder, MutableLong counter, ListMarshalledProvenance lmp) {
338        ListProvenanceProto.Builder lmpBuilder = ListProvenanceProto.newBuilder();
339
340        int curIndex = counter.intValue();
341        lmpBuilder.setIndex(curIndex);
342        counter.increment();
343
344        for (FlatMarshalledProvenance fmp : lmp) {
345            lmpBuilder.addValues(dispatchFMP(builder,counter,fmp));
346        }
347
348        builder.addLmp(lmpBuilder.build());
349        return curIndex;
350    }
351
352    /**
353     * Encodes the MapMarshalledProvenance into the RootProvenanceProto.
354     * @param builder The proto builder.
355     * @param counter The index counter.
356     * @param mmp The provenance to encode.
357     * @return The index of this MapMarshalledProvenance
358     */
359    private static int encodeMMP(RootProvenanceProto.Builder builder, MutableLong counter, MapMarshalledProvenance mmp) {
360        MapProvenanceProto.Builder mmpBuilder = MapProvenanceProto.newBuilder();
361
362        int curIndex = counter.intValue();
363        mmpBuilder.setIndex(curIndex);
364        counter.increment();
365
366        for (Pair<String,FlatMarshalledProvenance> p : mmp) {
367            mmpBuilder.putValues(p.getA(),dispatchFMP(builder,counter,p.getB()));
368        }
369
370        builder.addMmp(mmpBuilder.build());
371        return curIndex;
372    }
373
374    @Override
375    public String serializeToString(List<ObjectMarshalledProvenance> marshalledProvenances) {
376        RootProvenanceProto proto = serializeToProto(marshalledProvenances);
377
378        if (textFormat) {
379            return proto.toString();
380        } else {
381            return base64Encoder.encodeToString(proto.toByteArray());
382        }
383    }
384
385    @Override
386    public void serializeToFile(List<ObjectMarshalledProvenance> marshalledProvenances, Path path) throws IOException {
387        RootProvenanceProto proto = serializeToProto(marshalledProvenances);
388
389        if (textFormat) {
390            try (PrintWriter writer = new PrintWriter(Files.newBufferedWriter(path))) {
391                writer.println(proto.toString());
392            }
393        } else {
394            try (BufferedOutputStream bos = new BufferedOutputStream(Files.newOutputStream(path))) {
395                proto.writeTo(bos);
396            }
397        }
398    }
399}