001    
002    /*
003     * Copyright (C) 2012 Archie L. Cobbs. All rights reserved.
004     *
005     * $Id: PersistentObject.java 306 2012-03-07 03:27:21Z archie.cobbs $
006     */
007    
008    package org.dellroad.stuff.pobj;
009    
010    import java.io.BufferedInputStream;
011    import java.io.BufferedOutputStream;
012    import java.io.File;
013    import java.io.FileInputStream;
014    import java.io.FileOutputStream;
015    import java.io.IOException;
016    import java.util.ArrayList;
017    import java.util.HashSet;
018    import java.util.Set;
019    import java.util.concurrent.ExecutorService;
020    import java.util.concurrent.Executors;
021    import java.util.concurrent.ScheduledExecutorService;
022    import java.util.concurrent.ScheduledFuture;
023    import java.util.concurrent.TimeUnit;
024    
025    import javax.validation.ConstraintViolation;
026    import javax.xml.transform.stream.StreamResult;
027    import javax.xml.transform.stream.StreamSource;
028    
029    import org.dellroad.stuff.io.HardLink;
030    import org.slf4j.Logger;
031    import org.slf4j.LoggerFactory;
032    
033    /**
034     * Main class for Simple XML Persistence Objects (POBJ).
035     *
036     * <h3>Overview</h3>
037     *
038     * <p>
039     * Instances model an in-memory "database" represented by a root Java object and the graph of other objects that
040     * it references. The object graph is backed by a persistent XML file, which is read at initialization time and
041     * re-written after each change.
042     *
043     * <p>
044     * Changes are applied "wholesale" to the entire object graph, and are serialized and atomic. In other words, the
045     * entire object graph is read from, and written to, this class by value. As a result, it is not possible to change
046     * only a portion of the "database". The entire object graph is read and written as one thing. Similarly, the
047     * persistent XML file is updated by writing out a new, temporary copy and renaming the copy onto the original,
048     * using {@link File#renameTo File.renameTo()} for atomicity (on systems that support it, e.g., UNIX variants).
049     *
050     * <h3>Update Details</h3>
051     *
052     * <p>
053     * When the object graph is updated, it must pass validation checks, and then the persistent XML file is updated and
054     * listener notifications are sent out. Support for delayed write-back of the persistent XML file is included: this
055     * allows modifications that occur in rapid succession to be consolidated into a single filesystem write operation.
056     *
057     * <p>
058     * Support for optimistic locking is included. There is a "current version" number which is incremented each
059     * time the object graph is updated; writes may optionally specify this number to ensure no intervening changes
060     * have occurred. If concurrent updates are expected, applications may choose to implement a 3-way merge algorithm
061     * of some kind to handle optimistic locking failures.
062     *
063     * <p>
064     * Instances can be configured to preserve one or more backup copies of the persistent file on systems that support
065     * hard links (requires <a href="https://github.com/twall/jna">JNA</a>; see {@link HardLink}).
066     * Set the {@link #getNumBackups numBackups} property to enable.
067     *
068     * <h3>"Out-of-band" Writes</h3>
069     *
070     * <p>
071     * When a non-zero {@linkplain #getCheckInterval check interval} is configured, instances support "out-of-band" writes
072     * to the XML persistent file by some other process. This can be handy in cases where the other process (perhaps hand edits)
073     * is updating the persistent file and you want to have a running process pick up the changes just as if
074     * {@link PersistentObject#setRoot setRoot()} had been invoked. In particular, instances will detect the appearance
075     * of a new persistent file after an instance has started without one. In all cases, persistent objects must properly validate.
076     *
077     * <p>
078     * A special case of this is effected when {@link PersistentObject#setRoot setRoot()} is never explicitly invoked
079     * by the application. Then some other process must be responsible for all database updates, and this class automatically
080     * picks them up, validates them, and send out notifications.
081     *
082     * <h3>Empty Starts</h3>
083     *
084     * <p>
085     * An "empty start" occurs when an instance is {@linkplain #start started} but the persistent XML file is either missing,
086     * does not validate, or cannot be read for some other reason. In such cases, the instance will start with no object graph,
087     * and {@link #getRoot} will initially return null. This situation will correct itself as soon as the object graph is written
088     * via {@link #setRoot setRoot()} or the persistent file appears (effecting an "out-of-band" update).
089     *
090     * <p>
091     * Whther empty starts are allowed is determined by the {@link #isAllowEmptyStart allowEmptyStart} property (default
092     * {@code false}). When empty starts are disallowed, then {@link #start} will instead throw a {@link PersistentObjectException}.
093     * In this configuration, {@link #getRoot} can be relied upon to always return a non-null, valid root.
094     *
095     * <h3>Delegate Function</h3>
096     *
097     * <p>
098     * Instances must be configured with a {@link PersistentObjectDelegate} that knows how to validate the object graph
099     * and perform conversions to and from XML. See {@link PersistentObjectDelegate} and its implementations for details.
100     *
101     * <h3>Schema Changes</h3>
102     *
103     * <p>
104     * Like any database, the XML schema may evolve over time. The {@link PersistentObjectSchemaUpdater} class provides a simple
105     * way to apply and manage schema updates using XSLT transforms.
106     *
107     * @param <T> type of the root persistent object
108     * @see PersistentObjectDelegate
109     */
110    public class PersistentObject<T> {
111    
112        private static final long EXECUTOR_SHUTDOWN_TIMEOUT = 1000;             // 1 second
113    
114        protected final Logger log = LoggerFactory.getLogger(this.getClass());
115    
116        private final HashSet<PersistentObjectListener<T>> listeners = new HashSet<PersistentObjectListener<T>>();
117        private final PersistentObjectDelegate<T> delegate;
118        private final File persistentFile;
119        private final long writeDelay;
120        private final long checkInterval;
121    
122        private T root;
123        private T sharedRoot;
124        private int numBackups;
125        private ScheduledExecutorService scheduledExecutor;
126        private ExecutorService notifyExecutor;
127        private ScheduledFuture pendingWrite;
128        private long version;
129        private long timestamp;
130        private boolean allowEmptyStart;
131        private boolean started;
132    
133        /**
134         * Constructor.
135         *
136         * <p>
137         * The {@code writeDelay} is the maximum delay after an update operation before a write-back to the persistent file
138         * must be initiated.
139         *
140         * @param delegate delegate supplying required operations
141         * @param file the file used to persist
142         * @param writeDelay write delay in milliseconds, or zero for immediate write-back
143         * @param checkInterval check interval in milliseconds, or zero to disable persistent file checks
144         * @throws IllegalArgumentException if {@code delegate} or {@code file} is null
145         * @throws IllegalArgumentException if {@code writeDelay} or {@code checkInterval} is negative
146         */
147        public PersistentObject(PersistentObjectDelegate<T> delegate, File file, long writeDelay, long checkInterval) {
148            if (delegate == null)
149                throw new IllegalArgumentException("null delegate");
150            if (file == null)
151                throw new IllegalArgumentException("null file");
152            if (writeDelay < 0)
153                throw new IllegalArgumentException("negative writeDelay");
154            if (checkInterval < 0)
155                throw new IllegalArgumentException("negative checkInterval");
156            this.delegate = delegate;
157            this.persistentFile = file;
158            this.writeDelay = writeDelay;
159            this.checkInterval = checkInterval;
160        }
161    
162        /**
163         * Simplified constructor configuring for immediate write-back and no persistent file checks.
164         *
165         * <p>
166         * Equivalent to:
167         * <blockquote><code>
168         *  PersistentObject(delegate, file, 0L, 0L);
169         * </code></blockquote>
170         */
171        public PersistentObject(PersistentObjectDelegate<T> delegate, File file) {
172            this(delegate, file, 0, 0);
173        }
174    
175        /**
176         * Get the persistent file containing the XML form of the persisted object.
177         */
178        public File getPersistentFile() {
179            return this.persistentFile;
180        }
181    
182        /**
183         * Get the maximum delay after an update operation before a write-back to the persistent file
184         * must be initiated.
185         *
186         * @return write delay in milliseconds, or zero for immediate write-back
187         */
188        public long getWriteDelay() {
189            return this.writeDelay;
190        }
191    
192        /**
193         * Get the delay time between periodic checks for changes in the underlying persistent file.
194         *
195         * @return check interval in milliseconds, or zero if periodic checks are disabled
196         */
197        public long getCheckInterval() {
198            return this.checkInterval;
199        }
200    
201        /**
202         * Get the version of the current root.
203         *
204         * <p>
205         * This returns a value which increases monotonically with each update.
206         * The version number is not persisted with the persistent file; each instance of this class keeps
207         * its own version count. The version is reset to zero when {@link #stop stop()} is invoked.
208         *
209         * @return the current object version, or zero if no value has been loaded yet
210         */
211        public synchronized long getVersion() {
212            return this.version;
213        }
214    
215        /**
216         * Get the number of backup copies to preserve.
217         *
218         * <p>
219         * Backup files have suffixes of the form <code>.1</code>,  <code>.2</code>, etc.,
220         * in reverse chronological order. Each time a new root object is written, the existing files are rotated.
221         *
222         * <p>
223         * Back-ups are created via hard links and are only supported on UNIX systems.
224         *
225         * <p>
226         * The default is zero, which disables backups.
227         */
228        public int getNumBackups() {
229            return this.numBackups;
230        }
231    
232        /**
233         * Set the number of backup copies to preserve.
234         *
235         * @throws IllegalArgumentException if {@code numBackups} is negative
236         * @see #getNumBackups
237         */
238        public void setNumBackups(int numBackups) {
239            if (numBackups < 0)
240                throw new IllegalArgumentException("negative numBackups");
241            this.numBackups = numBackups;
242        }
243    
244        /**
245         * Determine whether this instance should allow an "empty start".
246         *
247         * <p>
248         * The default for this property is false.
249         */
250        public boolean isAllowEmptyStart() {
251            return this.allowEmptyStart;
252        }
253    
254        /**
255         * Configure whether an "empty start" is allowed.
256         *
257         * <p>
258         * The default for this property is false.
259         */
260        public void setAllowEmptyStart(boolean allowEmptyStart) {
261            this.allowEmptyStart = allowEmptyStart;
262        }
263    
264        /**
265         * Determine whether this instance is started.
266         */
267        public synchronized boolean isStarted() {
268            return this.started;
269        }
270    
271        /**
272         * Start this instance. Does nothing if already started.
273         *
274         * @throws PersistentObjectException if an error occurs
275         */
276        public synchronized void start() {
277    
278            // Already started?
279            if (this.started)
280                return;
281    
282            // Create executor services
283            this.scheduledExecutor = Executors.newSingleThreadScheduledExecutor();
284            this.notifyExecutor = Executors.newSingleThreadExecutor();
285    
286            // Read file (if it exists)
287            this.log.info(this + ": starting");
288            if (this.persistentFile.exists()) {
289                try {
290                    this.applyFile(this.persistentFile.lastModified());
291                } catch (PersistentObjectException e) {
292                    if (!this.isAllowEmptyStart())
293                        throw e;
294                    this.log.warn("empty start: unable to load persistent file `" + this.persistentFile + "': " + e);
295                }
296            } else {
297                if (!this.isAllowEmptyStart())
298                    throw new PersistentObjectException("persistent file `" + this.persistentFile + "' does not exist");
299                this.log.info(this + ": empty start: persistent file `" + this.persistentFile + "' does not exist");
300            }
301    
302            // Start checking the file
303            if (this.checkInterval > 0) {
304                this.scheduledExecutor.scheduleWithFixedDelay(new Runnable() {
305                    @Override
306                    public void run() {
307                        PersistentObject.this.checkFileTimeout();
308                    }
309                }, this.checkInterval, this.checkInterval, TimeUnit.MILLISECONDS);
310            }
311    
312            // Done
313            this.started = true;
314        }
315    
316        /**
317         * Stop this instance. Does nothing if already stopped.
318         *
319         * @throws PersistentObjectException if a delayed write back is pending and error occurs while performing the write
320         */
321        public synchronized void stop() {
322    
323            // Already stopped?
324            if (!this.started)
325                return;
326    
327            // Perform any lingering pending save now
328            if (this.cancelPendingWrite())
329                this.write(this.root);
330    
331            // Stop executor services
332            this.log.info(this + ": shutting down");
333            this.scheduledExecutor.shutdown();
334            this.notifyExecutor.shutdown();
335            this.awaitTermination(this.scheduledExecutor, "scheduledExecutor");
336            this.awaitTermination(this.notifyExecutor, "notifyExecutor");
337    
338            // Reset
339            this.scheduledExecutor = null;
340            this.notifyExecutor = null;
341            this.root = null;
342            this.version = 0;
343            this.timestamp = 0;
344            this.started = false;
345        }
346    
347        /**
348         * Atomically read the root object.
349         *
350         * <p>
351         * If there is no persistent file and no value has been set, null will be returned.
352         * However the persistent file may appear "out of band" at any time; if so, this will
353         * be detected within the configured {@linkplain #getCheckInterval check interval}.
354         *
355         * <p>
356         * This returns a deep copy of the current root object; any subsequent modifications are not written back.
357         *
358         * @return the current root instance, or null if after an "empty start"
359         * @throws IllegalStateException if this instance is not started
360         * @throws PersistentObjectException if an error occurs
361         */
362        public synchronized T getRoot() {
363    
364            // Sanity check
365            if (!this.started)
366                throw new IllegalStateException("not started");
367    
368            // Copy root
369            return this.root != null ? this.delegate.copy(this.root) : null;
370        }
371    
372        /**
373         * Get a shared copy of the root object.
374         *
375         * <p>
376         * This returns a copy of the root object, but it returns the same copy each time until the next change.
377         * This method is more efficient than {@link #getRoot}, but all callers must agree not to modify the returned object
378         * or any object in its graph of references.
379         *
380         * @return shared copy of the root instance, or null if after an "empty start"
381         */
382        public synchronized T getSharedRoot() {
383            if (this.sharedRoot == null)
384                this.sharedRoot = this.getRoot();
385            return this.sharedRoot;
386        }
387    
388        /**
389         * Atomically update the root object.
390         *
391         * <p>
392         * The given object is deep-copied and the copy replaces the current root.
393         *
394         * <p>
395         * If {@code expectedVersion} is non-zero, then if the current version is not equal to it,
396         * a {@link PersistentObjectVersionException} exception is thrown. This mechanism
397         * can be used for optimistic locking.
398         *
399         * @param newRoot new persistent object
400         * @param expectedVersion expected current version number, or zero to ignore the current version number
401         * @throws IllegalArgumentException if {@code newRoot} is null
402         * @throws IllegalArgumentException if {@code version} is negative
403         * @throws IllegalStateException if this instance is not started
404         * @throws PersistentObjectException if an error occurs
405         * @throws PersistentObjectVersionException if {@code version} is non-zero and not equal to the current version
406         * @throws PersistentObjectValidationException if the new root has validation errors
407         */
408        public final synchronized void setRoot(T newRoot, long expectedVersion) {
409            this.setRootInternal(newRoot, expectedVersion, false);
410        }
411    
412        @SuppressWarnings("unchecked")
413        private synchronized void setRootInternal(T newRoot, long expectedVersion, boolean readingFile) {
414    
415            // Sanity check
416            if (newRoot == null)
417                throw new IllegalArgumentException("null newRoot");
418            if (!this.started && !readingFile)
419                throw new IllegalStateException("not started");
420            if (expectedVersion < 0)
421                throw new IllegalStateException("negative expectedVersion");
422    
423            // Check version number
424            if (expectedVersion != 0 && this.version != expectedVersion)
425                throw new PersistentObjectVersionException(this.version, expectedVersion);
426    
427            // Check for sameness
428            if (this.root != null && this.delegate.isSameGraph(this.root, newRoot))
429                return;
430    
431            // Validate the new root
432            Set<ConstraintViolation<T>> violations = this.delegate.validate(newRoot);
433            if (!violations.isEmpty())
434                throw new PersistentObjectValidationException((Set<ConstraintViolation<?>>)(Object)violations);
435    
436            // Do the update
437            final T oldRoot = this.root;
438            this.root = this.delegate.copy(newRoot);
439            this.version++;
440            this.sharedRoot = null;
441    
442            // Perform write-back, either now or later
443            if (!readingFile) {
444                if (this.writeDelay == 0)
445                    this.write(this.root);
446                else if (this.pendingWrite == null) {
447                    this.pendingWrite = this.scheduledExecutor.schedule(new Runnable() {
448                        @Override
449                        public void run() {
450                            PersistentObject.this.writeTimeout();
451                        }
452                    }, this.writeDelay, TimeUnit.MILLISECONDS);
453                }
454            }
455    
456            // Notify listeners
457            this.notifyListeners(this.version, oldRoot, newRoot);
458        }
459    
460        /**
461         * Atomically update the root object.
462         *
463         * <p>
464         * The is a convenience method, equivalent to:
465         *  <blockquote>
466         *  <code>{@link #setRoot(Object, long) setRoot}(newRoot, 0)</code>
467         *  </blockquote>
468         *
469         * <p>
470         * This method cannot throw {@link PersistentObjectVersionException}.
471         */
472        public final synchronized void setRoot(T newRoot) {
473            this.setRoot(newRoot, 0);
474        }
475    
476        /**
477         * Check the persistent file for an "out-of-band" update.
478         *
479         * <p>
480         * If the persistent file has a newer timestamp than the timestamp of the most recently read
481         * or written version, then it will be read and applied to this instance.
482         *
483         * @throws IllegalStateException if this instance is not started
484         * @throws PersistentObjectException if an error occurs
485         */
486        public synchronized void checkFile() {
487    
488            // Sanity check
489            if (!this.started)
490                throw new IllegalStateException("not started");
491    
492            // Get file timestamp
493            long fileTime = this.persistentFile.lastModified();
494            if (fileTime == 0)
495                return;
496    
497            // Check whether file has newly appeared or just been updated
498            if (this.timestamp != 0 && fileTime <= this.timestamp)
499                return;
500    
501            // Read new file
502            this.log.info(this + ": detected out-of-band update of persistent file `" + this.persistentFile + "'");
503            this.applyFile(fileTime);
504        }
505    
506        /**
507         * Add a listener to be notified each time the object graph changes.
508         *
509         * @throws IllegalArgumentException if {@code listener} is null
510         */
511        public void addListener(PersistentObjectListener<T> listener) {
512            if (listener == null)
513                throw new IllegalArgumentException("null listener");
514            synchronized (this.listeners) {
515                this.listeners.add(listener);
516            }
517        }
518    
519        /**
520         * Remove a listener added via {@link #addListener addListener()}.
521         */
522        public void removeListener(PersistentObjectListener<T> listener) {
523            synchronized (this.listeners) {
524                this.listeners.remove(listener);
525            }
526        }
527    
528        /**
529         * Get a simple string description of this instance. This description appears in all log messages.
530         */
531        @Override
532        public String toString() {
533            return this.getClass().getSimpleName() + "[" + this.persistentFile.getName() + "]";
534        }
535    
536        /**
537         * Get the configured {@link PersistentObjectDelegate}.
538         */
539        protected PersistentObjectDelegate getDelegate() {
540            return this.delegate;
541        }
542    
543        /**
544         * Read the persistent file.
545         *
546         * @throws PersistentObjectException if an error occurs
547         */
548        protected T read() {
549    
550            // Open file
551            this.log.info(this + ": reading persistent file `" + this.persistentFile + "'");
552            BufferedInputStream input;
553            try {
554                input = new BufferedInputStream(new FileInputStream(this.persistentFile));
555            } catch (IOException e) {
556                throw new PersistentObjectException("error opening persistent file", e);
557            }
558    
559            // Parse XML
560            T obj;
561            try {
562                StreamSource source = new StreamSource(input);
563                source.setSystemId(this.persistentFile);
564                try {
565                    obj = this.delegate.deserialize(source);
566                } catch (IOException e) {
567                    throw new PersistentObjectException("error reading persistent file", e);
568                }
569            } finally {
570                try {
571                    input.close();
572                } catch (IOException e) {
573                    // ignore
574                }
575            }
576    
577            // Check result
578            if (obj == null)
579                throw new PersistentObjectException("null object returned by delegate.deserialize()");
580    
581            // Done
582            return obj;
583        }
584    
585        /**
586         * Write the persistent file and rotate any backups.
587         *
588         * <p>
589         * A temporary file is created in the same directory and then renamed to provide for an atomic update
590         * (on supporting operating systems).
591         *
592         * @throws IllegalArgumentException if {@code obj} is null
593         * @throws PersistentObjectException if an error occurs
594         */
595        protected final synchronized void write(T obj) {
596    
597            // Sanity check
598            if (obj == null)
599                throw new IllegalArgumentException("null obj");
600    
601            // Create temporary file
602            this.log.info(this + ": writing persistent file `" + this.persistentFile + "'");
603            File tempFile;
604            try {
605                tempFile = File.createTempFile(this.persistentFile.getName(), null, this.persistentFile.getParentFile());
606            } catch (IOException e) {
607                throw new PersistentObjectException("error creating temporary file", e);
608            }
609            try {
610    
611                // Open temporary file
612                BufferedOutputStream output;
613                try {
614                    output = new BufferedOutputStream(new FileOutputStream(tempFile));
615                } catch (IOException e) {
616                    throw new PersistentObjectException("error opening to temporary file", e);
617                }
618    
619                // Serialize to XML
620                try {
621                    StreamResult result = new StreamResult(output);
622                    result.setSystemId(tempFile);
623                    try {
624                        this.delegate.serialize(obj, result);
625                    } catch (IOException e) {
626                        throw new PersistentObjectException("error writing persistent file", e);
627                    }
628                    try {
629                        output.close();
630                    } catch (IOException e) {
631                        throw new PersistentObjectException("error closing temporary file", e);
632                    }
633                    output = null;
634                } finally {
635                    try {
636                        if (output != null)
637                            output.close();
638                    } catch (IOException e) {
639                        // ignore
640                    }
641                }
642    
643                // Get new modification time (prior to the rename, to avoid a race condition)
644                long newTimestamp = tempFile.lastModified();
645    
646                // Rotate backups
647                for (int i = this.numBackups - 1; i >= 0; i--) {
648                    File src = i > 0 ? new File(this.persistentFile.toString() + "." + i) : this.persistentFile;
649                    File dest = new File(this.persistentFile.toString() + "." + (i + 1));
650                    if (i == 0) {
651                        try {
652                            HardLink.link(src, dest);
653                        } catch (IOException e) {
654                            this.log.warn("unable to backup persistent file to `" + dest + "': " + e);
655                        }
656                    } else
657                        src.renameTo(dest);
658                }
659    
660                // Rename file
661                if (!tempFile.renameTo(this.persistentFile)) {
662                    throw new PersistentObjectException("error renaming temporary file `"
663                      + tempFile.getName() + "' to `" + this.persistentFile.getName() + "'");
664                }
665                tempFile = null;
666    
667                // Update file timestamp
668                this.timestamp = newTimestamp;
669            } finally {
670                if (tempFile != null)
671                    tempFile.delete();
672            }
673        }
674    
675        /**
676         * Notify listeners of a change in value.
677         *
678         * @param newVersion the version number associated with the new root
679         */
680        protected void notifyListeners(long newVersion, T oldRoot, T newRoot) {
681    
682            // Snapshot listeners
683            final ArrayList<PersistentObjectListener<T>> listenersCopy = new ArrayList<PersistentObjectListener<T>>();
684            synchronized (this.listeners) {
685                listenersCopy.addAll(this.listeners);
686            }
687    
688            // Notify them
689            final PersistentObjectEvent<T> event = new PersistentObjectEvent<T>(this, newVersion, oldRoot, newRoot);
690            this.notifyExecutor.submit(new Runnable() {
691                @Override
692                public void run() {
693                    PersistentObject.this.doNotifyListeners(listenersCopy, event);
694                }
695            });
696        }
697    
698        // Read the persistent file and apply it
699        private synchronized void applyFile(long newTimestamp) {
700            this.cancelPendingWrite();
701            this.timestamp = newTimestamp;                              // update timestamp even if update fails to avoid loops
702            this.setRootInternal(this.read(), 0, true);
703        }
704    
705        // Handle a write-back timeout
706        private synchronized void writeTimeout() {
707    
708            // Check for cancel race
709            if (this.pendingWrite == null)
710                return;
711            this.pendingWrite = null;
712    
713            // Write it
714            try {
715                this.write(this.root);
716            } catch (ThreadDeath t) {
717                throw t;
718            } catch (Throwable t) {
719                this.delegate.handleWritebackException(this, t);
720            }
721        }
722    
723        // Handle a check file timeout
724        private synchronized void checkFileTimeout() {
725    
726            // Handle race condition
727            if (!this.started)
728                return;
729    
730            // Check file
731            try {
732                this.checkFile();
733            } catch (ThreadDeath t) {
734                throw t;
735            } catch (Throwable t) {
736                this.log.error(this + ": error attempting to apply out-of-band update", t);
737            }
738        }
739    
740        // Cancel a pending write and return true if there was one
741        private synchronized boolean cancelPendingWrite() {
742            if (this.pendingWrite == null)
743                return false;
744            this.pendingWrite.cancel(false);
745            this.pendingWrite = null;
746            return true;
747        }
748    
749        // Notify listeners. This is invoked in a separate thread.
750        private void doNotifyListeners(ArrayList<PersistentObjectListener<T>> list, PersistentObjectEvent<T> event) {
751            for (PersistentObjectListener<T> listener : list) {
752                try {
753                    listener.handleEvent(event);
754                } catch (ThreadDeath t) {
755                    throw t;
756                } catch (Throwable t) {
757                    this.log.error(this + ": error notifying listener " + listener, t);
758                }
759            }
760        }
761    
762        // Wait for an ExecutorService to completely shut down
763        private void awaitTermination(ExecutorService executor, String name) {
764            boolean shutdown = false;
765            try {
766                shutdown = executor.awaitTermination(EXECUTOR_SHUTDOWN_TIMEOUT, TimeUnit.MILLISECONDS);
767            } catch (InterruptedException e) {
768                this.log.warn(this + ": interrupted while awaiting " + name + " termination");
769            }
770            if (!shutdown)
771                this.log.warn(this + ": failed to completely shut down " + name);
772        }
773    }
774