Updated storage to no longer try to compress the data, and to correctly use input/output streams from the RandomAccessFile. Also updated the tests for storage

Dorkbox-Util/src/dorkbox/util/storage/DiskStorage.java

@@ -34,7 +34,6 @@ import java.util.concurrent.locks.ReentrantLock;
  * Be wary of opening the database file in different JVM instances. Even with file-locks, you can corrupt the data.
  */
 @SuppressWarnings("unused")
-public
 class DiskStorage implements Storage {
     private final DelayTimer timer;
     private final ByteArrayWrapper defaultKey;
@@ -59,18 +58,17 @@ class DiskStorage implements Storage {
             this.timer = null;
         }
         else {
-            this.timer = new DelayTimer("Storage Writer", false, new DelayTimer.Callback() {
+            this.timer = new DelayTimer("Storage Writer", false, new Runnable() {
                 @Override
                 public
-                void execute() {
+                void run() {
-                    Map<ByteArrayWrapper, Object> actions = DiskStorage.this.actionMap;
-
                     ReentrantLock actionLock2 = DiskStorage.this.actionLock;
 
+                    Map<ByteArrayWrapper, Object> actions;
                     try {
                         actionLock2.lock();
-
                         // do a fast swap on the actionMap.
+                        actions = DiskStorage.this.actionMap;
                         DiskStorage.this.actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
                     } finally {
                         actionLock2.unlock();
@@ -207,7 +205,7 @@ class DiskStorage implements Storage {
         Object source = get0(key);
 
         if (source == null) {
-            // returned was null, so we should take value as the default
+            // returned was null, so we should save the default value
             putAndSave(key, data);
             return data;
         }
@@ -412,6 +410,7 @@ class DiskStorage implements Storage {
             throw new RuntimeException("Unable to act on closed storage");
         }
 
+        //noinspection SimplifiableIfStatement
         if (timer != null) {
             return this.timer.isWaiting();
         }

Dorkbox-Util/src/dorkbox/util/storage/MemoryStorage.java

@@ -22,9 +22,8 @@ import java.io.IOException;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
- *
+ * Storage that is in memory only (and is not persisted to disk)
  */
-public
 class MemoryStorage implements Storage {
     private final ConcurrentHashMap<ByteArrayWrapper, Object> storage;
     private final ByteArrayWrapper defaultKey;

Dorkbox-Util/src/dorkbox/util/storage/Metadata.java

@@ -23,11 +23,11 @@ import dorkbox.util.bytes.ByteArrayWrapper;
 import java.io.ByteArrayOutputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.RandomAccessFile;
 import java.lang.ref.WeakReference;
 import java.nio.channels.FileLock;
-import java.util.zip.DeflaterOutputStream;
-import java.util.zip.InflaterInputStream;
 
 @SuppressWarnings("unused")
 public
@@ -81,7 +81,7 @@ class Metadata {
      */
     static
     long getMetaDataPointer(int position) {
-        return StorageBase.FILE_HEADERS_REGION_LENGTH + INDEX_ENTRY_LENGTH * position;
+        return StorageBase.FILE_HEADERS_REGION_LENGTH + ((long) INDEX_ENTRY_LENGTH) * position;
     }
 
     /**
@@ -135,8 +135,10 @@ class Metadata {
 
         FileLock lock = file.getChannel()
                             .lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true);
+
         file.seek(origHeaderKeyPointer);
         file.readFully(buf);
+
         lock.release();
 
         Metadata r = new Metadata(ByteArrayWrapper.wrap(buf));
@@ -145,10 +147,12 @@ class Metadata {
         long recordHeaderPointer = Metadata.getDataPointer(position);
         lock = file.getChannel()
                    .lock(origHeaderKeyPointer, KEY_SIZE, true);
+
         file.seek(recordHeaderPointer);
         r.dataPointer = file.readLong();
         r.dataCapacity = file.readInt();
         r.dataCount = file.readInt();
+
         lock.release();
 
         if (r.dataPointer == 0L || r.dataCapacity == 0L || r.dataCount == 0L) {
@@ -173,17 +177,19 @@ class Metadata {
 
         FileLock lock = file.getChannel()
                             .lock(recordHeaderPointer, POINTER_INFO_SIZE, false);
+
         file.seek(recordHeaderPointer);
         file.writeLong(this.dataPointer);
         file.writeInt(this.dataCapacity);
         file.writeInt(this.dataCount);
+
         lock.release();
     }
 
     /**
      * Move a record to the new INDEX
      */
-    void move(RandomAccessFile file, int newIndex) throws IOException {
+    void moveRecord(RandomAccessFile file, int newIndex) throws IOException {
         byte[] buf = new byte[KEY_SIZE];
 
         long origHeaderKeyPointer = Metadata.getMetaDataPointer(this.indexPosition);
@@ -192,6 +198,7 @@ class Metadata {
 
         file.seek(origHeaderKeyPointer);
         file.readFully(buf);
+
         lock.release();
 
         long newHeaderKeyPointer = Metadata.getMetaDataPointer(newIndex);
@@ -200,6 +207,7 @@ class Metadata {
 
         file.seek(newHeaderKeyPointer);
         file.write(buf);
+
         lock.release();
 
 //        System.err.println("updating ptr: " +  this.indexPosition + " -> " + newIndex + " @ " + newHeaderKeyPointer + "-" + (newHeaderKeyPointer+INDEX_ENTRY_LENGTH));
@@ -230,6 +238,7 @@ class Metadata {
         // save the data
         file.seek(position);
         file.write(data);
+
         lock.release();
 
         // update header pointer info
@@ -249,6 +258,7 @@ class Metadata {
                             .lock(this.dataPointer, this.dataCount, true);
         file.seek(this.dataPointer);
         file.readFully(buf);
+
         lock.release();
 //        Sys.printArray(buf, buf.length, false, 0);
 
@@ -260,30 +270,32 @@ class Metadata {
      */
 
     static
-    <T> T readData(SerializationManager serializationManager, InflaterInputStream inputStream) throws IOException {
+    <T> T readData(final SerializationManager serializationManager, final Input input) throws IOException {
-        Input input = new Input(inputStream, 1024); // read 1024 at a time
+        // this is to reset the internal buffer of 'input'
+        input.setInputStream(input.getInputStream());
+
         @SuppressWarnings("unchecked")
         T readObject = (T) serializationManager.readFullClassAndObject(null, input);
         return readObject;
     }
 
     /**
-     * Writes data to the end of the file (which is where the datapointer is at)
+     * Writes data to the end of the file (which is where the datapointer is at). This must be locked/released in calling methods!
      */
     static
-    void writeDataFast(final SerializationManager serializationManager,
+    int writeData(final SerializationManager serializationManager,
-                       final Object data,
+                  final Object data,
-                       final RandomAccessFile file,
+                  final Output output) throws IOException {
-                       final DeflaterOutputStream outputStream) throws IOException {
+
-        // HAVE TO LOCK BEFORE THIS IS CALLED! (AND FREE AFTERWARDS!)
+        output.clear();
-        Output output = new Output(outputStream, 1024); // write 1024 at a time
+
         serializationManager.writeFullClassAndObject(null, output, data);
         output.flush();
 
-        outputStream.flush(); // sync-flush is enabled, so the output stream will finish compressing data.
+        return (int) output.total();
     }
 
-    void writeData(ByteArrayOutputStream byteArrayOutputStream, RandomAccessFile file) throws IOException {
+    void writeDataRaw(ByteArrayOutputStream byteArrayOutputStream, RandomAccessFile file) throws IOException {
         this.dataCount = byteArrayOutputStream.size();
 
         FileLock lock = file.getChannel()
@@ -292,7 +304,8 @@ class Metadata {
         FileOutputStream out = new FileOutputStream(file.getFD());
         file.seek(this.dataPointer);
         byteArrayOutputStream.writeTo(out);
-        out.flush();
+
+
         lock.release();
     }
 

Dorkbox-Util/src/dorkbox/util/storage/Storage.java

@@ -61,28 +61,31 @@ interface Storage {
      * <p/>
      * This will check to see if there is an associated key for that data, if not - it will use data as the default
      *
-     * @param data The data that will hold the copy of the data from disk
+     * @param data This is the default value, and if there is no value with the key in the DB this default value will be saved.
      */
     <T> T getAndPut(T data) throws IOException;
 
     /**
      * Returns the saved data for the specified key.
      *
-     * @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
+     * @param key The key used to check if data already exists.
+     * @param data This is the default value, and if there is no value with the key in the DB this default value will be saved.
      */
     <T> T getAndPut(String key, T data) throws IOException;
 
     /**
      * Returns the saved data for the specified key.
      *
-     * @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
+     * @param key The key used to check if data already exists.
+     * @param data This is the default value, and if there is no value with the key in the DB this default value will be saved.
      */
     <T> T getAndPut(byte[] key, T data) throws IOException;
 
     /**
      * Returns the saved data for the specified key.
      *
-     * @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
+     * @param key The key used to check if data already exists.
+     * @param data This is the default value, and if there is no value with the key in the DB this default value will be saved.
      */
     @SuppressWarnings("unchecked")
     <T> T getAndPut(ByteArrayWrapper key, T data) throws IOException;

Dorkbox-Util/src/dorkbox/util/storage/StorageBase.java

@@ -16,24 +16,29 @@
 package dorkbox.util.storage;
 
 import com.esotericsoftware.kryo.KryoException;
+import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
 import dorkbox.util.SerializationManager;
 import dorkbox.util.bytes.ByteArrayWrapper;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.*;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
 import java.lang.ref.WeakReference;
+import java.nio.channels.Channels;
 import java.nio.channels.FileLock;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.locks.ReentrantLock;
-import java.util.zip.Deflater;
-import java.util.zip.DeflaterOutputStream;
-import java.util.zip.Inflater;
-import java.util.zip.InflaterInputStream;
 
 
 // a note on file locking between c and java
@@ -42,7 +47,6 @@ import java.util.zip.InflaterInputStream;
 
 
 @SuppressWarnings("unused")
-public
 class StorageBase {
     private final Logger logger = LoggerFactory.getLogger(getClass().getSimpleName());
 
@@ -70,8 +74,9 @@ class StorageBase {
     private final ReentrantLock referenceLock = new ReentrantLock();
 
 
+    // file/raf that are used
     private final File baseFile;
-    private final RandomAccessFile file;
+    private final RandomAccessFile randomAccessFile;
 
 
     /**
@@ -93,11 +98,12 @@ class StorageBase {
     // save references to these, so they don't have to be created/destroyed any time there is I/O
     private final SerializationManager serializationManager;
 
-    private final Deflater deflater;
+    private final Output output;
-    private final DeflaterOutputStream outputStream;
+    private final Input input;
+
+    // input/output write buffer size before flushing to/from the file
+    public static final int BUFFER_SIZE = 1024;
 
-    private final Inflater inflater;
-    private final InflaterInputStream inputStream;
 
     /**
      * Creates or opens a new database file.
@@ -109,65 +115,75 @@ class StorageBase {
 
         this.baseFile = filePath;
 
-        File parentFile = this.baseFile.getParentFile();
+        boolean newStorage = !filePath.exists();
-        if (parentFile != null && !parentFile.exists()) {
+
-            if (!parentFile.mkdirs()) {
+        if (newStorage) {
-                throw new IOException("Unable to create dirs for: " + filePath);
+            File parentFile = this.baseFile.getParentFile();
+            if (parentFile != null && !parentFile.exists()) {
+                if (!parentFile.mkdirs()) {
+                    throw new IOException("Unable to create dirs for: " + filePath);
+                }
             }
         }
 
-        this.file = new RandomAccessFile(this.baseFile, "rw");
+        this.randomAccessFile = new RandomAccessFile(this.baseFile, "rw");
 
-        if (this.file.length() > FILE_HEADERS_REGION_LENGTH) {
+
-            this.file.seek(VERSION_HEADER_LOCATION);
+        if (newStorage || this.randomAccessFile.length() <= FILE_HEADERS_REGION_LENGTH) {
-            this.databaseVersion = this.file.readInt();
+            setVersion(this.randomAccessFile, 0);
-            this.numberOfRecords = this.file.readInt();
+            setRecordCount(this.randomAccessFile, 0);
-            this.dataPosition = this.file.readLong();
+
+            // pad the metadata with 21 records, so there is about 1k of padding before the data starts
+            long indexPointer = Metadata.getMetaDataPointer(21);
+            setDataStartPosition(indexPointer);
+            // have to make sure we can read header info (even if it's blank)
+            this.randomAccessFile.setLength(indexPointer);
         }
         else {
-            setVersionNumber(this.file, 0);
+            this.randomAccessFile.seek(VERSION_HEADER_LOCATION);
+            this.databaseVersion = this.randomAccessFile.readInt();
+            this.numberOfRecords = this.randomAccessFile.readInt();
+            this.dataPosition = this.randomAccessFile.readLong();
 
-            // always start off with 4 records
+            if (this.randomAccessFile.length() < this.dataPosition) {
-            setRecordCount(this.file, 4);
+                this.logger.error("Corrupted storage file!");
-
+                throw new IllegalArgumentException("Unable to parse header information from storage. Maybe it's corrupted?");
-            long indexPointer = Metadata.getMetaDataPointer(4);
+            }
-            setDataPosition(this.file, indexPointer);
-            // have to make sure we can read header info (even if it's blank)
-            this.file.setLength(indexPointer);
-        }
-
-        if (this.file.length() < this.dataPosition) {
-            this.logger.error("Corrupted storage file!");
-            throw new IllegalArgumentException("Unable to parse header information from storage. Maybe it's corrupted?");
         }
 
         //noinspection AutoBoxing
         this.logger.info("Storage version: {}", this.databaseVersion);
 
-        this.deflater = new Deflater(7, true);
-        FileOutputStream fileOutputStream = new FileOutputStream(this.file.getFD());
-        this.outputStream = new DeflaterOutputStream(fileOutputStream, this.deflater, 65536);
 
-        this.inflater = new Inflater(true);
+        // If we want to use compression (no need really, since this file is small already),
-        this.inputStream = new InflaterInputStream(new FileInputStream(this.file.getFD()), this.inflater, 65536);
+        // then we have to make sure it's sync'd on flush AND have actually call outputStream.flush().
+        final InputStream inputStream = Channels.newInputStream(randomAccessFile.getChannel());
+        final OutputStream outputStream = Channels.newOutputStream(randomAccessFile.getChannel());
+
+        // read/write 1024 bytes at a time
+        output = new Output(outputStream, BUFFER_SIZE);
+        input = new Input(inputStream, BUFFER_SIZE);
+
 
         //noinspection AutoBoxing
         this.weight = 0.5F;
         this.memoryIndex = new ConcurrentHashMap<ByteArrayWrapper, Metadata>(this.numberOfRecords);
 
-        Metadata meta;
+        if (!newStorage) {
-        for (int index = 0; index < this.numberOfRecords; index++) {
+            Metadata meta;
-            meta = Metadata.readHeader(this.file, index);
+            for (int index = 0; index < this.numberOfRecords; index++) {
-            if (meta == null) {
+                meta = Metadata.readHeader(this.randomAccessFile, index);
-                // because we guarantee that empty metadata are AWLAYS at the end of the section, if we get a null one, break!
+                if (meta == null) {
-                break;
+                    // because we guarantee that empty metadata are ALWAYS at the end of the section, if we get a null one, break!
+                    break;
+                }
+                this.memoryIndex.put(meta.key, meta);
             }
-            this.memoryIndex.put(meta.key, meta);
-        }
 
-        if (this.memoryIndex.size() != this.numberOfRecords) {
+            if (this.memoryIndex.size() != this.numberOfRecords) {
-            setRecordCount(this.file, this.memoryIndex.size());
+                setRecordCount(this.randomAccessFile, this.memoryIndex.size());
-            this.logger.warn("Mismatch record count in storage, auto-correcting size.");
+                this.logger.warn("Mismatch record count in storage, auto-correcting size.");
+            }
         }
     }
 
@@ -255,11 +271,12 @@ class StorageBase {
 
 
         try {
-            // else, we have to load it from disk
+//            System.err.println("--Reading data from: " + meta.dataPointer);
-            this.inflater.reset();
-            this.file.seek(meta.dataPointer);
 
-            T readRecordData = Metadata.readData(this.serializationManager, this.inputStream);
+            // else, we have to load it from disk
+            this.randomAccessFile.seek(meta.dataPointer);
+
+            T readRecordData = Metadata.readData(this.serializationManager, this.input);
 
             if (readRecordData != null) {
                 // now stuff it into our reference cache for future lookups!
@@ -310,13 +327,16 @@ class StorageBase {
     void close() {
         // pending ops flushed (protected by lock)
         // not protected by lock
+
+        this.logger.info("Closing storage file: '{}'", this.baseFile.getAbsolutePath());
+
         try {
-            this.file.getFD()
+            this.randomAccessFile.getFD()
-                     .sync();
+                                 .sync();
-            this.file.close();
+            this.input.close();
+            this.randomAccessFile.close();
             this.memoryIndex.clear();
 
-            this.inputStream.close();
         } catch (IOException e) {
             this.logger.error("Error while closing the file", e);
         }
@@ -330,7 +350,7 @@ class StorageBase {
     long getFileSize() {
         // protected by actionLock
         try {
-            return this.file.length();
+            return this.randomAccessFile.length();
         } catch (IOException e) {
             this.logger.error("Error getting file size for {}", this.baseFile.getAbsolutePath(), e);
             return -1L;
@@ -355,9 +375,7 @@ class StorageBase {
      * Will also save the object in a cache.
      */
     private
-    void save0(ByteArrayWrapper key, Object object, DeflaterOutputStream fileOutputStream, Deflater deflater) {
+    void save0(ByteArrayWrapper key, Object object) {
-        deflater.reset();
-
         Metadata metaData = this.memoryIndex.get(key);
         int currentRecordCount = this.numberOfRecords;
 
@@ -367,29 +385,31 @@ class StorageBase {
                 if (currentRecordCount == 1) {
                     // if we are the ONLY one, then we can do things differently.
                     // just dump the data again to disk.
-                    FileLock lock = this.file.getChannel()
+                    FileLock lock = this.randomAccessFile.getChannel()
-                                             .lock(this.dataPosition,
+                                                         .lock(this.dataPosition,
                                                    Long.MAX_VALUE - this.dataPosition,
                                                    false); // don't know how big it is, so max value it
-                    this.file.seek(this.dataPosition); // this is the end of the file, we know this ahead-of-time
+
-                    Metadata.writeDataFast(this.serializationManager, object, file, fileOutputStream);
+                    this.randomAccessFile.seek(this.dataPosition); // this is the end of the file, we know this ahead-of-time
+                    Metadata.writeData(this.serializationManager, object, this.output);
                     // have to re-specify the capacity and size
                     //noinspection NumericCastThatLosesPrecision
-                    int sizeOfWrittenData = (int) (this.file.length() - this.dataPosition);
+                    int sizeOfWrittenData = (int) (this.randomAccessFile.length() - this.dataPosition);
 
                     metaData.dataCapacity = sizeOfWrittenData;
                     metaData.dataCount = sizeOfWrittenData;
+
                     lock.release();
                 }
                 else {
                     // this is comparatively slow, since we serialize it first to get the size, then we put it in the file.
-                    ByteArrayOutputStream dataStream = getDataAsByteArray(this.serializationManager, this.logger, object, deflater);
+                    ByteArrayOutputStream dataStream = getDataAsByteArray(this.serializationManager, this.logger, object);
 
                     int size = dataStream.size();
                     if (size > metaData.dataCapacity) {
                         deleteRecordData(metaData, size);
                         // stuff this record to the end of the file, since it won't fit in it's current location
-                        metaData.dataPointer = this.file.length();
+                        metaData.dataPointer = this.randomAccessFile.length();
                         // have to make sure that the CAPACITY of the new one is the SIZE of the new data!
                         // and since it is going to the END of the file, we do that.
                         metaData.dataCapacity = size;
@@ -398,45 +418,51 @@ class StorageBase {
 
                     // TODO: should check to see if the data is different. IF SO, then we write, otherwise nothing!
 
-                    metaData.writeData(dataStream, this.file);
+                    metaData.writeDataRaw(dataStream, this.randomAccessFile);
                 }
 
-                metaData.writeDataInfo(this.file);
+                metaData.writeDataInfo(this.randomAccessFile);
             } catch (IOException e) {
                 this.logger.error("Error while saving data to disk", e);
             }
         }
         else {
+            // metadata == null...
             try {
-                // try to move the read head in order
+                // set the number of records that this storage has
-                setRecordCount(this.file, currentRecordCount + 1);
+                setRecordCount(this.randomAccessFile, currentRecordCount + 1);
 
                 // This will make sure that there is room to write a new record. This is zero indexed.
                 // this will skip around if moves occur
-                ensureIndexCapacity(this.file);
+                ensureIndexCapacity(this.randomAccessFile);
 
                 // append record to end of file
-                long length = this.file.length();
+                long length = this.randomAccessFile.length();
+
+//                System.err.println("--Writing data to: " + length);
+
                 metaData = new Metadata(key, currentRecordCount, length);
-                metaData.writeMetaDataInfo(this.file);
+                metaData.writeMetaDataInfo(this.randomAccessFile);
 
                 // add new entry to the index
                 this.memoryIndex.put(key, metaData);
-                setRecordCount(this.file, currentRecordCount + 1);
 
                 // save out the data. Because we KNOW that we are writing this to the end of the file,
                 // there are some tricks we can use.
 
-                FileLock lock = this.file.getChannel()
+                // don't know how big it is, so max value it
-                                         .lock(length, Long.MAX_VALUE - length, false); // don't know how big it is, so max value it
+                FileLock lock = this.randomAccessFile.getChannel()
-                this.file.seek(length); // this is the end of the file, we know this ahead-of-time
+                                                     .lock(0, Long.MAX_VALUE, false);
-                Metadata.writeDataFast(this.serializationManager, object, file, fileOutputStream);
+
+                // this is the end of the file, we know this ahead-of-time
+                this.randomAccessFile.seek(length);
+
+                int total = Metadata.writeData(this.serializationManager, object, this.output);
                 lock.release();
 
-                metaData.dataCount = deflater.getTotalOut();
+                metaData.dataCount = metaData.dataCapacity = total;
-                metaData.dataCapacity = metaData.dataCount;
                 // have to save it.
-                metaData.writeDataInfo(this.file);
+                metaData.writeDataInfo(this.randomAccessFile);
             } catch (IOException e) {
                 this.logger.error("Error while writing data to disk", e);
                 return;
@@ -449,32 +475,31 @@ class StorageBase {
 
 
     private static
-    ByteArrayOutputStream getDataAsByteArray(SerializationManager serializationManager, Logger logger, Object data, Deflater deflater) throws IOException {
+    ByteArrayOutputStream getDataAsByteArray(SerializationManager serializationManager, Logger logger, Object data) throws IOException {
-        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-        OutputStream outputStream = new DeflaterOutputStream(byteArrayOutputStream, deflater);
         Output output = new Output(outputStream, 1024); // write 1024 at a time
+
         serializationManager.writeFullClassAndObject(logger, output, data);
         output.flush();
 
         outputStream.flush();
         outputStream.close();
 
-        return byteArrayOutputStream;
+        return outputStream;
     }
 
     void doActionThings(Map<ByteArrayWrapper, Object> actions) {
-        DeflaterOutputStream outputStream2 = this.outputStream;
-        Deflater deflater2 = this.deflater;
 
         // actions is thrown away after this invocation. GC can pick it up.
         // we are only interested in the LAST action that happened for some data.
         // items to be "autosaved" are automatically injected into "actions".
-        for (Entry<ByteArrayWrapper, Object> entry : actions.entrySet()) {
+        final Set<Entry<ByteArrayWrapper, Object>> entries = actions.entrySet();
+        for (Entry<ByteArrayWrapper, Object> entry : entries) {
             Object object = entry.getValue();
             ByteArrayWrapper key = entry.getKey();
 
             // our action list is for explicitly saving objects (but not necessarily "registering" them to be auto-saved
-            save0(key, object, outputStream2, deflater2);
+            save0(key, object);
         }
     }
 
@@ -501,11 +526,11 @@ class StorageBase {
 
     private
     void deleteRecordData(Metadata deletedRecord, int sizeOfDataToAdd) throws IOException {
-        if (this.file.length() == deletedRecord.dataPointer + deletedRecord.dataCapacity) {
+        if (this.randomAccessFile.length() == deletedRecord.dataPointer + deletedRecord.dataCapacity) {
             // shrink file since this is the last record in the file
-            FileLock lock = this.file.getChannel()
+            FileLock lock = this.randomAccessFile.getChannel()
-                                     .lock(deletedRecord.dataPointer, Long.MAX_VALUE - deletedRecord.dataPointer, false);
+                                                 .lock(deletedRecord.dataPointer, Long.MAX_VALUE - deletedRecord.dataPointer, false);
-            this.file.setLength(deletedRecord.dataPointer);
+            this.randomAccessFile.setLength(deletedRecord.dataPointer);
             lock.release();
         }
         else {
@@ -527,11 +552,11 @@ class StorageBase {
 
                 if (endIndexPointer < this.dataPosition && endIndexPointer <= newEndOfDataPointer) {
                     // one option is to shrink the RECORD section to fit the new data
-                    setDataPosition(this.file, newEndOfDataPointer);
+                    setDataStartPosition(newEndOfDataPointer);
                 }
                 else {
                     // option two is to grow the RECORD section, and put the data at the end of the file
-                    setDataPosition(this.file, endOfDataPointer);
+                    setDataStartPosition(endOfDataPointer);
                 }
             }
             else {
@@ -539,7 +564,7 @@ class StorageBase {
                 if (previous != null) {
                     // append space of deleted record onto previous record
                     previous.dataCapacity += deletedRecord.dataCapacity;
-                    previous.writeDataInfo(this.file);
+                    previous.writeDataInfo(this.randomAccessFile);
                 }
                 else {
                     // because there is no "previous", that means we MIGHT be the FIRST record
@@ -556,15 +581,15 @@ class StorageBase {
         int currentNumRecords = this.memoryIndex.size();
 
         if (deleteRecord.indexPosition != currentNumRecords - 1) {
-            Metadata last = Metadata.readHeader(this.file, currentNumRecords - 1);
+            Metadata last = Metadata.readHeader(this.randomAccessFile, currentNumRecords - 1);
             assert last != null;
 
-            last.move(this.file, deleteRecord.indexPosition);
+            last.moveRecord(this.randomAccessFile, deleteRecord.indexPosition);
         }
 
         this.memoryIndex.remove(key);
 
-        setRecordCount(this.file, currentNumRecords - 1);
+        setRecordCount(this.randomAccessFile, currentNumRecords - 1);
     }
 
 
@@ -572,13 +597,14 @@ class StorageBase {
      * Writes the number of records header to the file.
      */
     private
-    void setVersionNumber(RandomAccessFile file, int versionNumber) throws IOException {
+    void setVersion(RandomAccessFile file, int versionNumber) throws IOException {
         this.databaseVersion = versionNumber;
 
-        FileLock lock = this.file.getChannel()
+        FileLock lock = this.randomAccessFile.getChannel()
-                                 .lock(VERSION_HEADER_LOCATION, 4, false);
+                                             .lock(VERSION_HEADER_LOCATION, 4, false);
         file.seek(VERSION_HEADER_LOCATION);
         file.writeInt(versionNumber);
+
         lock.release();
     }
 
@@ -587,26 +613,34 @@ class StorageBase {
      */
     private
     void setRecordCount(RandomAccessFile file, int numberOfRecords) throws IOException {
-        this.numberOfRecords = numberOfRecords;
+        if (this.numberOfRecords != numberOfRecords) {
+            this.numberOfRecords = numberOfRecords;
 
-        FileLock lock = this.file.getChannel()
+//            System.err.println("Set recordCount: " + numberOfRecords);
-                                 .lock(NUM_RECORDS_HEADER_LOCATION, 4, false);
+
-        file.seek(NUM_RECORDS_HEADER_LOCATION);
+            FileLock lock = this.randomAccessFile.getChannel()
-        file.writeInt(numberOfRecords);
+                                                 .lock(NUM_RECORDS_HEADER_LOCATION, 4, false);
-        lock.release();
+            file.seek(NUM_RECORDS_HEADER_LOCATION);
+            file.writeInt(numberOfRecords);
+
+            lock.release();
+        }
     }
 
     /**
      * Writes the data start position to the file.
      */
     private
-    void setDataPosition(RandomAccessFile file, long dataPositionPointer) throws IOException {
+    void setDataStartPosition(long dataPositionPointer) throws IOException {
-        this.dataPosition = dataPositionPointer;
+        FileLock lock = this.randomAccessFile.getChannel()
+                                             .lock(DATA_START_HEADER_LOCATION, 8, false);
+
+//        System.err.println("Setting data position: " + dataPositionPointer);
+        dataPosition = dataPositionPointer;
+
+        randomAccessFile.seek(DATA_START_HEADER_LOCATION);
+        randomAccessFile.writeLong(dataPositionPointer);
 
-        FileLock lock = this.file.getChannel()
-                                 .lock(DATA_START_HEADER_LOCATION, 8, false);
-        file.seek(DATA_START_HEADER_LOCATION);
-        file.writeLong(dataPositionPointer);
         lock.release();
     }
 
@@ -616,7 +650,7 @@ class StorageBase {
 
     void setVersion(int versionNumber) {
         try {
-            setVersionNumber(this.file, versionNumber);
+            setVersion(this.randomAccessFile, versionNumber);
         } catch (IOException e) {
             this.logger.error("Unable to set the version number", e);
         }
@@ -650,14 +684,16 @@ class StorageBase {
      */
     private
     void ensureIndexCapacity(RandomAccessFile file) throws IOException {
-        int numberOfRecords = this.numberOfRecords; // because we are zero indexed, this is ALSO the index where the record will START
+        // because we are zero indexed, this is ALSO the index where the record will START
-        long endIndexPointer = Metadata.getMetaDataPointer(
+        int numberOfRecords = this.numberOfRecords;
-                        numberOfRecords + 1); // +1 because this is where that index will END (the start of the NEXT one)
+
+        // +1 because this is where that index will END (the start of the NEXT one)
+        long endIndexPointer = Metadata.getMetaDataPointer(numberOfRecords + 1);
 
         // just set the data position to the end of the file, since we don't have any data yet.
         if (endIndexPointer > file.length() && numberOfRecords == 0) {
             file.setLength(endIndexPointer);
-            setDataPosition(file, endIndexPointer);
+            setDataStartPosition(endIndexPointer);
             return;
         }
 
@@ -675,13 +711,21 @@ class StorageBase {
         int newNumberOfRecords = getWeightedNewRecordCount(numberOfRecords);
         endIndexPointer = Metadata.getMetaDataPointer(newNumberOfRecords);
 
+
         // sometimes the endIndexPointer is in the middle of data, so we cannot move a record to where
         // data already exists, we have to move it to the end. Since we GUARANTEE that there is never "free space" at the
         // end of a file, this is ok
-        endIndexPointer = Math.max(endIndexPointer, file.length());
+        if (endIndexPointer > file.length()) {
+            // make sure we adjust the file size
+            file.setLength(endIndexPointer);
+        }
+        else {
+            endIndexPointer = file.length();
+        }
 
         // we know that the start of the NEW data position has to be here.
-        setDataPosition(file, endIndexPointer);
+        setDataStartPosition(endIndexPointer);
+
 
         long writeDataPosition = endIndexPointer;
 

Dorkbox-Util/src/dorkbox/util/storage/Store.java

@@ -32,6 +32,21 @@ class Store {
     @SuppressWarnings("SpellCheckingInspection")
     private static final Map<File, Storage> storages = new HashMap<File, Storage>(1);
 
+    // Make sure that the timer is run on shutdown. A HARD shutdown will just POW! kill it, a "nice" shutdown will run the hook
+    private static Thread shutdownHook = new Thread(new Runnable() {
+        @Override
+        public
+        void run() {
+            Store.shutdown();
+        }
+    });
+
+    static {
+        // add a shutdown hook to make sure that we properly flush/shutdown storage.
+        Runtime.getRuntime()
+               .addShutdownHook(shutdownHook);
+    }
+
     public static
     DiskMaker Disk() {
         return new DiskMaker();

Dorkbox-Util/test/dorkbox/util/StorageTest.java

@@ -18,6 +18,13 @@ import java.util.Arrays;
 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
 public
 class StorageTest {
+    static int total = 10;
+    // the initial size is specified during disk.storage construction, and is based on the number of padded records.
+    private static final long initialSize = 1024L;
+
+    // this is the size for each record (determined by looking at the output when writing the file)
+    private static final int sizePerRecord = 23;
+
 
     private static final File TEST_DB = new File("sampleFile.records");
     private static final SerializationManager manager = new SerializationManager() {
@@ -126,7 +133,7 @@ class StorageTest {
         long size1 = storage.getFileSize();
 
         Assert.assertEquals("count is not correct", numberOfRecords1, 0);
-        Assert.assertEquals("size is not correct", size1, 208L);  // NOTE this will change based on the data size added!
+        Assert.assertEquals("size is not correct", size1, initialSize);
 
         Store.close(storage);
 
@@ -134,6 +141,7 @@ class StorageTest {
                        .file(TEST_DB)
                        .serializer(manager)
                        .make();
+
         int numberOfRecords2 = storage.size();
         long size2 = storage.getFileSize();
 
@@ -147,13 +155,12 @@ class StorageTest {
     @Test
     public
     void testAddAsOne() throws IOException, ClassNotFoundException {
-        int total = 100;
-
         try {
             Storage storage = Store.Disk()
                                    .file(TEST_DB)
                                    .serializer(manager)
                                    .make();
+
             for (int i = 0; i < total; i++) {
                 add(storage, i);
             }
@@ -163,6 +170,7 @@ class StorageTest {
                            .file(TEST_DB)
                            .serializer(manager)
                            .make();
+
             for (int i = 0; i < total; i++) {
                 String record1Data = createData(i);
                 String readRecord = readRecord(storage, i);
@@ -177,16 +185,20 @@ class StorageTest {
         }
     }
 
+    /**
+     * Adds data to storage using the SAME key each time (so each entry is overwritten).
+     * @throws IOException
+     * @throws ClassNotFoundException
+     */
     @Test
     public
     void testAddNoKeyRecords() throws IOException, ClassNotFoundException {
-        int total = 100;
-
         try {
             Storage storage = Store.Disk()
                                    .file(TEST_DB)
                                    .serializer(manager)
                                    .make();
+
             for (int i = 0; i < total; i++) {
                 log("adding record " + i + "...");
                 String addRecord = createData(i);
@@ -208,13 +220,15 @@ class StorageTest {
             log("reading record " + (total - 1) + "...");
             String readData = storage.get();
 
+            // the ONLY entry in storage should be the last one that we added
             Assert.assertEquals("Object is not the same", dataCheck, readData);
 
             int numberOfRecords1 = storage.size();
             long size1 = storage.getFileSize();
 
             Assert.assertEquals("count is not correct", numberOfRecords1, 1);
-            Assert.assertEquals("size is not correct", size1, 235L); // NOTE this will change based on the data size added!
+
+            Assert.assertEquals("size is not correct", size1, initialSize + sizePerRecord);
 
             Store.close(storage);
         } catch (Exception e) {
@@ -226,13 +240,12 @@ class StorageTest {
     @Test
     public
     void testAddRecords_DelaySaveA() throws IOException, ClassNotFoundException {
-        int total = 100;
-
         try {
             Storage storage = Store.Disk()
                                    .file(TEST_DB)
                                    .serializer(manager)
                                    .make();
+
             for (int i = 0; i < total; i++) {
                 add(storage, i);
             }
@@ -272,13 +285,12 @@ class StorageTest {
     @Test
     public
     void testAddRecords_DelaySaveB() throws IOException, ClassNotFoundException {
-        int total = 100;
-
         try {
             Storage storage = Store.Disk()
                                    .file(TEST_DB)
                                    .serializer(manager)
                                    .make();
+
             for (int i = 0; i < total; i++) {
                 add(storage, i);
             }
@@ -296,6 +308,7 @@ class StorageTest {
                            .file(TEST_DB)
                            .serializer(manager)
                            .make();
+
             for (int i = 0; i < total; i++) {
                 String dataCheck = createData(i);
                 String readRecord = readRecord(storage, i);
@@ -313,13 +326,12 @@ class StorageTest {
     @Test
     public
     void testLoadRecords() throws IOException, ClassNotFoundException {
-        int total = 100;
-
         try {
             Storage storage = Store.Disk()
                                    .file(TEST_DB)
                                    .serializer(manager)
                                    .make();
+
             for (int i = 0; i < total; i++) {
                 String addRecord = add(storage, i);
                 String readRecord = readRecord(storage, i);
@@ -332,6 +344,7 @@ class StorageTest {
                            .file(TEST_DB)
                            .serializer(manager)
                            .make();
+
             for (int i = 0; i < total; i++) {
                 String dataCheck = createData(i);
                 String readRecord = readRecord(storage, i);
@@ -346,8 +359,8 @@ class StorageTest {
 
             storage.put(createKey, data);
 
-            Data data2 = new Data();
+            Data data2;
-            storage.getAndPut(createKey, data2);
+            data2 = storage.getAndPut(createKey, new Data());
             Assert.assertEquals("Object is not the same", data, data2);
 
             Store.close(storage);
@@ -356,8 +369,7 @@ class StorageTest {
                            .serializer(manager)
                            .make();
 
-            data2 = new Data();
+            data2 = storage.getAndPut(createKey, new Data());
-            storage.getAndPut(createKey, data2);
             Assert.assertEquals("Object is not the same", data, data2);
 
             Store.close(storage);
@@ -371,13 +383,16 @@ class StorageTest {
     @Test
     public
     void testAddRecordsDelete1Record() throws IOException, ClassNotFoundException {
-        int total = 100;
+        if (total < 4) {
+            throw new IOException("Unable to run test with too few entries.");
+        }
 
         try {
             Storage storage = Store.Disk()
                                    .file(TEST_DB)
                                    .serializer(manager)
                                    .make();
+
             for (int i = 0; i < total; i++) {
                 String addRecord = add(storage, i);
                 String readRecord = readRecord(storage, i);
@@ -390,6 +405,7 @@ class StorageTest {
                            .file(TEST_DB)
                            .serializer(manager)
                            .make();
+
             for (int i = 0; i < total; i++) {
                 String dataCheck = createData(i);
                 String readRecord = readRecord(storage, i);
@@ -442,13 +458,12 @@ class StorageTest {
     @Test
     public
     void testUpdateRecords() throws IOException, ClassNotFoundException {
-        int total = 100;
-
         try {
             Storage storage = Store.Disk()
                                    .file(TEST_DB)
                                    .serializer(manager)
                                    .make();
+
             for (int i = 0; i < total; i++) {
                 String addRecord = add(storage, i);
                 String readRecord = readRecord(storage, i);
@@ -507,13 +522,12 @@ class StorageTest {
     @Test
     public
     void testSaveAllRecords() throws IOException, ClassNotFoundException {
-        int total = 100;
-
         try {
             Storage storage = Store.Disk()
                                    .file(TEST_DB)
                                    .serializer(manager)
                                    .make();
+
             for (int i = 0; i < total; i++) {
                 Data data = new Data();
                 makeData(data);
@@ -533,8 +547,8 @@ class StorageTest {
             for (int i = 0; i < total; i++) {
                 String createKey = createKey(i);
 
-                Data data2 = new Data();
+                Data data2;
-                storage.getAndPut(createKey, data2);
+                data2 = storage.getAndPut(createKey, new Data());
                 Assert.assertEquals("Object is not the same", data, data2);
             }
             Store.close(storage);
@@ -607,19 +621,19 @@ class StorageTest {
 
         data.strings = new String[] {"ab012", "", null, "!@#$", "<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>"};
         data.ints = new int[] {-1234567, 1234567, -1, 0, 1, Integer.MAX_VALUE, Integer.MIN_VALUE};
-        data.shorts = new short[] {-12345, 12345, -1, 0, 1, Short.MAX_VALUE, Short.MIN_VALUE};
+        data.shorts = new short[] {(short) -12345, (short) 12345, (short) -1, (short) 0, (short) 1, Short.MAX_VALUE, Short.MIN_VALUE};
         data.floats = new float[] {0, -0, 1, -1, 123456, -123456, 0.1f, 0.2f, -0.3f, (float) Math.PI, Float.MAX_VALUE, Float.MIN_VALUE};
         data.doubles = new double[] {0, -0, 1, -1, 123456, -123456, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE, Double.MIN_VALUE};
-        data.longs = new long[] {0, -0, 1, -1, 123456, -123456, 99999999999l, -99999999999l, Long.MAX_VALUE, Long.MIN_VALUE};
+        data.longs = new long[] {0, -0, 1, -1, 123456, -123456, 99999999999L, -99999999999L, Long.MAX_VALUE, Long.MIN_VALUE};
-        data.bytes = new byte[] {-123, 123, -1, 0, 1, Byte.MAX_VALUE, Byte.MIN_VALUE};
+        data.bytes = new byte[] {(byte) -123, (byte) 123, (byte) -1, (byte) 0, (byte) 1, Byte.MAX_VALUE, Byte.MIN_VALUE};
         data.chars = new char[] {32345, 12345, 0, 1, 63, Character.MAX_VALUE, Character.MIN_VALUE};
         data.booleans = new boolean[] {true, false};
         data.Ints = new Integer[] {-1234567, 1234567, -1, 0, 1, Integer.MAX_VALUE, Integer.MIN_VALUE};
         data.Shorts = new Short[] {-12345, 12345, -1, 0, 1, Short.MAX_VALUE, Short.MIN_VALUE};
-        data.Floats = new Float[] {0f, -0f, 1f, -1f, 123456f, -123456f, 0.1f, 0.2f, -0.3f, (float) Math.PI, Float.MAX_VALUE,
+        data.Floats = new Float[] {0.0f, -0.0f, 1.0f, -1.0f, 123456.0f, -123456.0f, 0.1f, 0.2f, -0.3f, (float) Math.PI, Float.MAX_VALUE,
                                    Float.MIN_VALUE};
-        data.Doubles = new Double[] {0d, -0d, 1d, -1d, 123456d, -123456d, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE, Double.MIN_VALUE};
+        data.Doubles = new Double[] {0.0d, -0.0d, 1.0d, -1.0d, 123456.0d, -123456.0d, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE, Double.MIN_VALUE};
-        data.Longs = new Long[] {0l, -0l, 1l, -1l, 123456l, -123456l, 99999999999l, -99999999999l, Long.MAX_VALUE, Long.MIN_VALUE};
+        data.Longs = new Long[] {0L, -0L, 1L, -1L, 123456L, -123456L, 99999999999L, -99999999999L, Long.MAX_VALUE, Long.MIN_VALUE};
         data.Bytes = new Byte[] {-123, 123, -1, 0, 1, Byte.MAX_VALUE, Byte.MIN_VALUE};
         data.Chars = new Character[] {32345, 12345, 0, 1, 63, Character.MAX_VALUE, Character.MIN_VALUE};
         data.Booleans = new Boolean[] {true, false};