Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.store.jdbc;
018
019import java.io.IOException;
020import java.sql.SQLException;
021import java.util.Arrays;
022import java.util.LinkedList;
023
024import org.apache.activemq.ActiveMQMessageAudit;
025import org.apache.activemq.broker.ConnectionContext;
026import org.apache.activemq.command.ActiveMQDestination;
027import org.apache.activemq.command.Message;
028import org.apache.activemq.command.MessageAck;
029import org.apache.activemq.command.MessageId;
030import org.apache.activemq.command.XATransactionId;
031import org.apache.activemq.store.AbstractMessageStore;
032import org.apache.activemq.store.IndexListener;
033import org.apache.activemq.store.MessageRecoveryListener;
034import org.apache.activemq.util.ByteSequence;
035import org.apache.activemq.util.ByteSequenceData;
036import org.apache.activemq.util.IOExceptionSupport;
037import org.apache.activemq.wireformat.WireFormat;
038import org.slf4j.Logger;
039import org.slf4j.LoggerFactory;
040
041/**
042 * 
043 */
044public class JDBCMessageStore extends AbstractMessageStore {
045
046    class Duration {
047        static final int LIMIT = 100;
048        final long start = System.currentTimeMillis();
049        final String name;
050
051        Duration(String name) {
052            this.name = name;
053        }
054        void end() {
055            end(null);
056        }
057        void end(Object o) {
058            long duration = System.currentTimeMillis() - start;
059
060            if (duration > LIMIT) {
061                System.err.println(name + " took a long time: " + duration + "ms " + o);
062            }
063        }
064    }
065    private static final Logger LOG = LoggerFactory.getLogger(JDBCMessageStore.class);
066    protected final WireFormat wireFormat;
067    protected final JDBCAdapter adapter;
068    protected final JDBCPersistenceAdapter persistenceAdapter;
069    protected ActiveMQMessageAudit audit;
070    protected final LinkedList<Long> pendingAdditions = new LinkedList<Long>();
071    final long[] perPriorityLastRecovered = new long[10];
072
073    public JDBCMessageStore(JDBCPersistenceAdapter persistenceAdapter, JDBCAdapter adapter, WireFormat wireFormat, ActiveMQDestination destination, ActiveMQMessageAudit audit) throws IOException {
074        super(destination);
075        this.persistenceAdapter = persistenceAdapter;
076        this.adapter = adapter;
077        this.wireFormat = wireFormat;
078        this.audit = audit;
079
080        if (destination.isQueue() && persistenceAdapter.getBrokerService().shouldRecordVirtualDestination(destination)) {
081            recordDestinationCreation(destination);
082        }
083        resetBatching();
084    }
085
086    private void recordDestinationCreation(ActiveMQDestination destination) throws IOException {
087        TransactionContext c = persistenceAdapter.getTransactionContext();
088        try {
089            c = persistenceAdapter.getTransactionContext();
090            if (adapter.doGetLastAckedDurableSubscriberMessageId(c, destination, destination.getQualifiedName(), destination.getQualifiedName()) < 0) {
091                adapter.doRecordDestination(c, destination);
092            }
093        } catch (SQLException e) {
094            JDBCPersistenceAdapter.log("JDBC Failure: ", e);
095            throw IOExceptionSupport.create("Failed to record destination: " + destination + ". Reason: " + e, e);
096        } finally {
097            c.close();
098        }
099    }
100
101    public void addMessage(final ConnectionContext context, final Message message) throws IOException {
102        MessageId messageId = message.getMessageId();
103        if (audit != null && audit.isDuplicate(message)) {
104            if (LOG.isDebugEnabled()) {
105                LOG.debug(destination.getPhysicalName()
106                    + " ignoring duplicated (add) message, already stored: "
107                    + messageId);
108            }
109            return;
110        }
111
112        // if xaXid present - this is a prepare - so we don't yet have an outcome
113        final XATransactionId xaXid =  context != null ? context.getXid() : null;
114
115        // Serialize the Message..
116        byte data[];
117        try {
118            ByteSequence packet = wireFormat.marshal(message);
119            data = ByteSequenceData.toByteArray(packet);
120        } catch (IOException e) {
121            throw IOExceptionSupport.create("Failed to broker message: " + messageId + " in container: " + e, e);
122        }
123
124        // Get a connection and insert the message into the DB.
125        TransactionContext c = persistenceAdapter.getTransactionContext(context);
126        long sequenceId;
127        synchronized (pendingAdditions) {
128            sequenceId = persistenceAdapter.getNextSequenceId();
129            final long sequence = sequenceId;
130            message.getMessageId().setEntryLocator(sequence);
131
132            if (xaXid == null) {
133                pendingAdditions.add(sequence);
134
135                c.onCompletion(new Runnable() {
136                    public void run() {
137                        // jdbc close or jms commit - while futureOrSequenceLong==null ordered
138                        // work will remain pending on the Queue
139                        message.getMessageId().setFutureOrSequenceLong(sequence);
140                    }
141                });
142
143                if (indexListener != null) {
144                    indexListener.onAdd(new IndexListener.MessageContext(context, message, new Runnable() {
145                        @Override
146                        public void run() {
147                            // cursor add complete
148                            synchronized (pendingAdditions) { pendingAdditions.remove(sequence); }
149                        }
150                    }));
151                } else {
152                    pendingAdditions.remove(sequence);
153                }
154            }
155        }
156        try {
157            adapter.doAddMessage(c, sequenceId, messageId, destination, data, message.getExpiration(),
158                    this.isPrioritizedMessages() ? message.getPriority() : 0, xaXid);
159        } catch (SQLException e) {
160            JDBCPersistenceAdapter.log("JDBC Failure: ", e);
161            throw IOExceptionSupport.create("Failed to broker message: " + messageId + " in container: " + e, e);
162        } finally {
163            c.close();
164        }
165        if (xaXid == null) {
166            onAdd(message, sequenceId, message.getPriority());
167        }
168    }
169
170    // jdbc commit order is random with concurrent connections - limit scan to lowest pending
171    private long minPendingSequeunceId() {
172        synchronized (pendingAdditions) {
173            if (!pendingAdditions.isEmpty()) {
174                return pendingAdditions.get(0);
175            } else {
176                // nothing pending, ensure scan is limited to current state
177                return persistenceAdapter.sequenceGenerator.getLastSequenceId() + 1;
178            }
179        }
180    }
181
182    @Override
183    public void updateMessage(Message message) throws IOException {
184        TransactionContext c = persistenceAdapter.getTransactionContext();
185        try {
186            adapter.doUpdateMessage(c, destination, message.getMessageId(), ByteSequenceData.toByteArray(wireFormat.marshal(message)));
187        } catch (SQLException e) {
188            JDBCPersistenceAdapter.log("JDBC Failure: ", e);
189            throw IOExceptionSupport.create("Failed to update message: " + message.getMessageId() + " in container: " + e, e);
190        } finally {
191            c.close();
192        }
193    }
194
195    protected void onAdd(Message message, long sequenceId, byte priority) {}
196
197    public void addMessageReference(ConnectionContext context, MessageId messageId, long expirationTime, String messageRef) throws IOException {
198        // Get a connection and insert the message into the DB.
199        TransactionContext c = persistenceAdapter.getTransactionContext(context);
200        try {
201            adapter.doAddMessageReference(c, persistenceAdapter.getNextSequenceId(), messageId, destination, expirationTime, messageRef);
202        } catch (SQLException e) {
203            JDBCPersistenceAdapter.log("JDBC Failure: ", e);
204            throw IOExceptionSupport.create("Failed to broker message: " + messageId + " in container: " + e, e);
205        } finally {
206            c.close();
207        }
208    }
209
210    public Message getMessage(MessageId messageId) throws IOException {
211        // Get a connection and pull the message out of the DB
212        TransactionContext c = persistenceAdapter.getTransactionContext();
213        try {
214            byte data[] = adapter.doGetMessage(c, messageId);
215            if (data == null) {
216                return null;
217            }
218
219            Message answer = (Message)wireFormat.unmarshal(new ByteSequence(data));
220            return answer;
221        } catch (IOException e) {
222            throw IOExceptionSupport.create("Failed to broker message: " + messageId + " in container: " + e, e);
223        } catch (SQLException e) {
224            JDBCPersistenceAdapter.log("JDBC Failure: ", e);
225            throw IOExceptionSupport.create("Failed to broker message: " + messageId + " in container: " + e, e);
226        } finally {
227            c.close();
228        }
229    }
230
231    public String getMessageReference(MessageId messageId) throws IOException {
232        long id = messageId.getBrokerSequenceId();
233
234        // Get a connection and pull the message out of the DB
235        TransactionContext c = persistenceAdapter.getTransactionContext();
236        try {
237            return adapter.doGetMessageReference(c, id);
238        } catch (IOException e) {
239            throw IOExceptionSupport.create("Failed to broker message: " + messageId + " in container: " + e, e);
240        } catch (SQLException e) {
241            JDBCPersistenceAdapter.log("JDBC Failure: ", e);
242            throw IOExceptionSupport.create("Failed to broker message: " + messageId + " in container: " + e, e);
243        } finally {
244            c.close();
245        }
246    }
247
248    public void removeMessage(ConnectionContext context, MessageAck ack) throws IOException {
249
250        long seq = ack.getLastMessageId().getFutureOrSequenceLong() != null ?
251                (Long) ack.getLastMessageId().getFutureOrSequenceLong() :
252                persistenceAdapter.getStoreSequenceIdForMessageId(context, ack.getLastMessageId(), destination)[0];
253
254        // Get a connection and remove the message from the DB
255        TransactionContext c = persistenceAdapter.getTransactionContext(context);
256        try {
257            adapter.doRemoveMessage(c, seq, context != null ? context.getXid() : null);
258        } catch (SQLException e) {
259            JDBCPersistenceAdapter.log("JDBC Failure: ", e);
260            throw IOExceptionSupport.create("Failed to broker message: " + ack.getLastMessageId() + " in container: " + e, e);
261        } finally {
262            c.close();
263        }
264    }
265
266    public void recover(final MessageRecoveryListener listener) throws Exception {
267
268        // Get all the Message ids out of the database.
269        TransactionContext c = persistenceAdapter.getTransactionContext();
270        try {
271            c = persistenceAdapter.getTransactionContext();
272            adapter.doRecover(c, destination, new JDBCMessageRecoveryListener() {
273                public boolean recoverMessage(long sequenceId, byte[] data) throws Exception {
274                    Message msg = (Message) wireFormat.unmarshal(new ByteSequence(data));
275                    msg.getMessageId().setBrokerSequenceId(sequenceId);
276                    return listener.recoverMessage(msg);
277                }
278
279                public boolean recoverMessageReference(String reference) throws Exception {
280                    return listener.recoverMessageReference(new MessageId(reference));
281                }
282            });
283        } catch (SQLException e) {
284            JDBCPersistenceAdapter.log("JDBC Failure: ", e);
285            throw IOExceptionSupport.create("Failed to recover container. Reason: " + e, e);
286        } finally {
287            c.close();
288        }
289    }
290
291    /**
292     * @see org.apache.activemq.store.MessageStore#removeAllMessages(ConnectionContext)
293     */
294    public void removeAllMessages(ConnectionContext context) throws IOException {
295        // Get a connection and remove the message from the DB
296        TransactionContext c = persistenceAdapter.getTransactionContext(context);
297        try {
298            adapter.doRemoveAllMessages(c, destination);
299        } catch (SQLException e) {
300            JDBCPersistenceAdapter.log("JDBC Failure: ", e);
301            throw IOExceptionSupport.create("Failed to broker remove all messages: " + e, e);
302        } finally {
303            c.close();
304        }
305    }
306
307    @Override
308    public int getMessageCount() throws IOException {
309        int result = 0;
310        TransactionContext c = persistenceAdapter.getTransactionContext();
311        try {
312
313            result = adapter.doGetMessageCount(c, destination);
314
315        } catch (SQLException e) {
316            JDBCPersistenceAdapter.log("JDBC Failure: ", e);
317            throw IOExceptionSupport.create("Failed to get Message Count: " + destination + ". Reason: " + e, e);
318        } finally {
319            c.close();
320        }
321        return result;
322    }
323
324    /**
325     * @param maxReturned
326     * @param listener
327     * @throws Exception
328     * @see org.apache.activemq.store.MessageStore#recoverNextMessages(int,
329     *      org.apache.activemq.store.MessageRecoveryListener)
330     */
331    public void recoverNextMessages(int maxReturned, final MessageRecoveryListener listener) throws Exception {
332        TransactionContext c = persistenceAdapter.getTransactionContext();
333        try {
334            if (LOG.isTraceEnabled()) {
335                LOG.trace(this + " recoverNext lastRecovered:" + Arrays.toString(perPriorityLastRecovered) + ", minPending:" + minPendingSequeunceId());
336            }
337            adapter.doRecoverNextMessages(c, destination, perPriorityLastRecovered, minPendingSequeunceId(),
338                    maxReturned, isPrioritizedMessages(), new JDBCMessageRecoveryListener() {
339
340                public boolean recoverMessage(long sequenceId, byte[] data) throws Exception {
341                        Message msg = (Message)wireFormat.unmarshal(new ByteSequence(data));
342                        msg.getMessageId().setBrokerSequenceId(sequenceId);
343                        msg.getMessageId().setFutureOrSequenceLong(sequenceId);
344                        listener.recoverMessage(msg);
345                        trackLastRecovered(sequenceId, msg.getPriority());
346                        return true;
347                }
348
349                public boolean recoverMessageReference(String reference) throws Exception {
350                    if (listener.hasSpace()) {
351                        listener.recoverMessageReference(new MessageId(reference));
352                        return true;
353                    }
354                    return false;
355                }
356
357            });
358        } catch (SQLException e) {
359            JDBCPersistenceAdapter.log("JDBC Failure: ", e);
360        } finally {
361            c.close();
362        }
363
364    }
365
366    private void trackLastRecovered(long sequenceId, int priority) {
367        perPriorityLastRecovered[isPrioritizedMessages() ? priority : 0] = sequenceId;
368    }
369
370    /**
371     * @see org.apache.activemq.store.MessageStore#resetBatching()
372     */
373    public void resetBatching() {
374        if (LOG.isTraceEnabled()) {
375            LOG.trace(this + " resetBatching. last recovered: " + Arrays.toString(perPriorityLastRecovered));
376        }
377        setLastRecovered(-1);
378    }
379
380    private void setLastRecovered(long val) {
381        for (int i=0;i<perPriorityLastRecovered.length;i++) {
382            perPriorityLastRecovered[i] = val;
383        }
384    }
385
386
387    @Override
388    public void setBatch(MessageId messageId) {
389        if (LOG.isTraceEnabled()) {
390            LOG.trace(this + " setBatch: last recovered: " + Arrays.toString(perPriorityLastRecovered));
391        }
392        try {
393            long[] storedValues = persistenceAdapter.getStoreSequenceIdForMessageId(null, messageId, destination);
394            setLastRecovered(storedValues[0]);
395        } catch (IOException ignoredAsAlreadyLogged) {
396            resetBatching();
397        }
398        if (LOG.isTraceEnabled()) {
399            LOG.trace(this + " setBatch: new last recovered: " + Arrays.toString(perPriorityLastRecovered));
400        }
401    }
402
403
404    public void setPrioritizedMessages(boolean prioritizedMessages) {
405        super.setPrioritizedMessages(prioritizedMessages);
406    }
407
408    @Override
409    public String toString() {
410        return destination.getPhysicalName() + ",pendingSize:" + pendingAdditions.size();
411    }
412
413}