diff --git a/src/dorkbox/util/storage/DefaultStorageSerializationManager.java b/src/dorkbox/util/storage/DefaultStorageSerializationManager.java new file mode 100644 index 0000000..cecc2a0 --- /dev/null +++ b/src/dorkbox/util/storage/DefaultStorageSerializationManager.java @@ -0,0 +1,97 @@ +/* + * Copyright 2014 dorkbox, llc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dorkbox.util.storage; + +import java.io.IOException; + +import org.slf4j.Logger; + +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import com.esotericsoftware.minlog.Log; + +import dorkbox.util.SerializationManager; +import io.netty.buffer.ByteBuf; + +class DefaultStorageSerializationManager implements SerializationManager { + private Kryo kryo = new Kryo() {{ + // we don't want logging from Kryo... + Log.set(Log.LEVEL_ERROR); + }}; + + @Override + public + void register(final Class clazz) { + kryo.register(clazz); + } + + @Override + public + void register(final Class clazz, final Serializer serializer) { + kryo.register(clazz, serializer); + } + + @Override + public + void register(final Class type, final Serializer serializer, final int id) { + kryo.register(type, serializer, id); + } + + @Override + public + void write(final ByteBuf buffer, final Object message) { + final Output output = new Output(); + writeFullClassAndObject(null, output, message); + buffer.writeBytes(output.getBuffer()); + } + + @Override + public + Object read(final ByteBuf buffer, final int length) throws IOException { + final Input input = new Input(); + buffer.readBytes(input.getBuffer()); + + final Object o = readFullClassAndObject(null, input); + buffer.skipBytes(input.position()); + + return o; + } + + @Override + public + void writeFullClassAndObject(final Logger logger, final Output output, final Object value) { + kryo.writeClassAndObject(output, value); + } + + @Override + public + Object readFullClassAndObject(final Logger logger, final Input input) throws IOException { + return kryo.readClassAndObject(input); + } + + @Override + public + void finishInit() { + } + + @Override + public + boolean initialized() { + return false; + } +} diff --git a/src/dorkbox/util/storage/DiskStorage.java b/src/dorkbox/util/storage/DiskStorage.java index 5d24a79..6c70b78 100644 --- a/src/dorkbox/util/storage/DiskStorage.java +++ b/src/dorkbox/util/storage/DiskStorage.java @@ -122,45 +122,37 @@ class DiskStorage implements Storage { /** * Reads a object using the default (blank) key, and casts it to the expected class - * - * @throws IOException if there is a problem deserializing data from disk */ @Override public final - T get() throws IOException { + T get() { return get0(this.defaultKey); } /** * Reads a object using the specific key, and casts it to the expected class - * - * @throws IOException if there is a problem deserializing data from disk */ @Override public final - T get(String key) throws IOException { + T get(String key) { return get0(new StorageKey(key)); } /** * Reads a object using the specific key, and casts it to the expected class - * - * @throws IOException if there is a problem deserializing data from disk */ @Override public final - T get(byte[] key) throws IOException { + T get(byte[] key) { return get0(new StorageKey(key)); } /** * Reads a object using the specific key, and casts it to the expected class - * - * @throws IOException if there is a problem deserializing data from disk */ @Override public final - T get(StorageKey key) throws IOException { + T get(StorageKey key) { return get0(key); } @@ -173,7 +165,7 @@ class DiskStorage implements Storage { */ @Override public - T getAndPut(T data) throws IOException { + T getAndPut(T data) { return getAndPut(this.defaultKey, data); } @@ -184,7 +176,7 @@ class DiskStorage implements Storage { */ @Override public - T getAndPut(String key, T data) throws IOException { + T getAndPut(String key, T data) { StorageKey wrap = new StorageKey(key); return getAndPut(wrap, data); @@ -197,19 +189,21 @@ class DiskStorage implements Storage { */ @Override public - T getAndPut(byte[] key, T data) throws IOException { + T getAndPut(byte[] key, T data) { return getAndPut(new StorageKey(key), data); } /** - * Returns the saved data for the specified key. Also saves the data. + * Returns the saved data (or null) for the specified key. Also saves the data. * * @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db) + * + * @return NULL if the saved data was the wrong type for the specified key. */ @Override @SuppressWarnings("unchecked") public - T getAndPut(StorageKey key, T data) throws IOException { + T getAndPut(StorageKey key, T data) { Object source = get0(key); if (source == null) { @@ -222,7 +216,16 @@ class DiskStorage implements Storage { final Class savedCLass = source.getClass(); if (!expectedClass.isAssignableFrom(savedCLass)) { - throw new IOException("Saved value type '" + source.getClass() + "' is different that expected value"); + String message = "Saved value type '" + savedCLass + "' is different than expected value '" + expectedClass + "'"; + + if (storage.logger != null) { + storage.logger.error(message); + } + else { + System.err.print(message); + } + + return null; } } @@ -231,11 +234,9 @@ class DiskStorage implements Storage { /** * Reads a object from pending or from storage - * - * @throws IOException if there is a problem deserializing data from disk */ private - T get0(StorageKey key) throws IOException { + T get0(StorageKey key) { if (!this.isOpen.get()) { throw new RuntimeException("Unable to act on closed storage"); } diff --git a/src/dorkbox/util/storage/MemoryStorage.java b/src/dorkbox/util/storage/MemoryStorage.java index 63c44f8..e8fb87e 100644 --- a/src/dorkbox/util/storage/MemoryStorage.java +++ b/src/dorkbox/util/storage/MemoryStorage.java @@ -16,7 +16,6 @@ package dorkbox.util.storage; import java.io.File; -import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; /** @@ -99,7 +98,7 @@ class MemoryStorage implements Storage { */ @Override public - T getAndPut(T data) throws IOException { + T getAndPut(T data) { return getAndPut(this.defaultKey, data); } @@ -110,7 +109,7 @@ class MemoryStorage implements Storage { */ @Override public - T getAndPut(String key, T data) throws IOException { + T getAndPut(String key, T data) { StorageKey wrap = new StorageKey(key); return getAndPut(wrap, data); @@ -123,14 +122,14 @@ class MemoryStorage implements Storage { */ @Override public - T getAndPut(byte[] key, T data) throws IOException { + T getAndPut(byte[] key, T data) { return getAndPut(new StorageKey(key), data); } @SuppressWarnings("unchecked") @Override public - T getAndPut(final StorageKey key, final T data) throws IOException { + T getAndPut(final StorageKey key, final T data) { final Object o = storage.get(key); if (o == null) { storage.put(key, data); diff --git a/src/dorkbox/util/storage/Storage.java b/src/dorkbox/util/storage/Storage.java index 329381e..705b534 100644 --- a/src/dorkbox/util/storage/Storage.java +++ b/src/dorkbox/util/storage/Storage.java @@ -16,11 +16,11 @@ package dorkbox.util.storage; import java.io.File; -import java.io.IOException; /** * */ +@SuppressWarnings("unused") public interface Storage { /** @@ -36,22 +36,22 @@ interface Storage { /** * Reads a object using the DEFAULT key ("") key, and casts it to the expected class */ - T get() throws IOException; + T get(); + + /** + * Reads a object using the specific key, and casts it to the expected class. + */ + T get(String key); /** * Reads a object using the specific key, and casts it to the expected class */ - T get(String key) throws IOException; + T get(byte[] key); /** * Reads a object using the specific key, and casts it to the expected class */ - T get(byte[] key) throws IOException; - - /** - * Reads a object using the specific key, and casts it to the expected class - */ - T get(StorageKey key) throws IOException; + T get(StorageKey key); /** * Uses the DEFAULT key ("") to return saved data. @@ -60,7 +60,7 @@ interface Storage { * * @param data This is the default value, and if there is no value with the key in the DB this default value will be saved. */ - T getAndPut(T data) throws IOException; + T getAndPut(T data); /** * Returns the saved data for the specified key. @@ -68,7 +68,7 @@ interface Storage { * @param key The key used to check if data already exists. * @param data This is the default value, and if there is no value with the key in the DB this default value will be saved. */ - T getAndPut(String key, T data) throws IOException; + T getAndPut(String key, T data); /** * Returns the saved data for the specified key. @@ -76,7 +76,7 @@ interface Storage { * @param key The key used to check if data already exists. * @param data This is the default value, and if there is no value with the key in the DB this default value will be saved. */ - T getAndPut(byte[] key, T data) throws IOException; + T getAndPut(byte[] key, T data); /** * Returns the saved data for the specified key. @@ -84,7 +84,7 @@ interface Storage { * @param key The key used to check if data already exists. * @param data This is the default value, and if there is no value with the key in the DB this default value will be saved. */ - T getAndPut(StorageKey key, T data) throws IOException; + T getAndPut(StorageKey key, T data); /** * Saves the given data to storage with the associated key. diff --git a/src/dorkbox/util/storage/StorageBase.java b/src/dorkbox/util/storage/StorageBase.java index e08f1b6..e45c041 100644 --- a/src/dorkbox/util/storage/StorageBase.java +++ b/src/dorkbox/util/storage/StorageBase.java @@ -33,7 +33,6 @@ import java.util.concurrent.locks.ReentrantLock; import org.slf4j.Logger; -import com.esotericsoftware.kryo.KryoException; import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; @@ -46,8 +45,9 @@ import dorkbox.util.SerializationManager; // Also, file locks on linux are ADVISORY. if an app doesn't care about locks, then it can do stuff -- even if locked by another app +@SuppressWarnings("unused") class StorageBase { - private final Logger logger; + protected final Logger logger; // File pointer to the data start pointer header. @@ -101,7 +101,7 @@ class StorageBase { private final Input input; // input/output write buffer size before flushing to/from the file - public static final int BUFFER_SIZE = 1024; + private static final int BUFFER_SIZE = 1024; /** @@ -249,10 +249,10 @@ class StorageBase { } /** - * @return an object for a specified key form referenceCache FIRST, then from DISK + * @return an object for a specified key form referenceCache FIRST, then from DISK. NULL if it doesn't exist or there was an error. */ final - T get(StorageKey key) throws IOException { + T get(StorageKey key) { // NOT protected by lock Metadata meta = this.memoryIndex.get(key); @@ -299,11 +299,14 @@ class StorageBase { return readRecordData; } catch (Exception e) { - if (e instanceof KryoException && e.getMessage().contains("(missing no-arg constructor)")) { - throw new IOException("Cannot get data from disk: " + e.getMessage().substring(0, e.getMessage().indexOf(OS.LINE_SEPARATOR))); - } else { - throw new IOException("Cannot get data from disk", e); + String message = e.getMessage().substring(0,e.getMessage().indexOf(OS.LINE_SEPARATOR)); + if (logger != null) { + logger.error("Error reading data from disk: {}", message); } + else { + System.err.print("Error reading data from disk: " + message); + } + return null; } } diff --git a/src/dorkbox/util/storage/StorageSystem.java b/src/dorkbox/util/storage/StorageSystem.java index 79cb0a2..eac2305 100644 --- a/src/dorkbox/util/storage/StorageSystem.java +++ b/src/dorkbox/util/storage/StorageSystem.java @@ -25,6 +25,7 @@ import org.slf4j.Logger; import org.slf4j.helpers.NOPLogger; import dorkbox.util.FileUtil; +import dorkbox.util.OS; import dorkbox.util.SerializationManager; public @@ -47,6 +48,7 @@ class StorageSystem { .addShutdownHook(shutdownHook); } + /** * Creates a persistent, on-disk storage system. Writes to disk are queued, so it is recommended to NOT edit/change an object after * it has been put into storage, or whenever it does changes, make sure to put it back into storage (to update the saved record) @@ -212,13 +214,13 @@ class StorageSystem { * Makes the storage system */ public - Storage make() { + Storage build() { if (this.file == null) { throw new IllegalArgumentException("file cannot be null!"); } if (this.serializationManager == null) { - throw new IllegalArgumentException("serializer cannot be null!"); + this.serializationManager = createDefaultSerializationManager(); } // if we load from a NEW storage at the same location as an ALREADY EXISTING storage, @@ -244,13 +246,24 @@ class StorageSystem { storage = new DiskStorage(this.file, this.serializationManager, this.readOnly, this.logger); storages.put(this.file, storage); } catch (IOException e) { - logger.error("Unable to open storage", e); + String message = e.getMessage().substring(0,e.getMessage().indexOf(OS.LINE_SEPARATOR)); + if (logger != null) { + logger.error("Unable to open storage file at {}. {}", this.file, message); + } + else { + System.err.print("Unable to open storage file at " + this.file + ". " + message); + } } } return storage; } } + + private + SerializationManager createDefaultSerializationManager() { + return new DefaultStorageSerializationManager(); + } } @@ -261,10 +274,10 @@ class StorageSystem { class MemoryMaker { /** - * Makes the storage system + * Builds the storage system */ public - MemoryStorage make() throws IOException { + MemoryStorage build() { return new MemoryStorage(); } } diff --git a/test/dorkbox/util/StorageTest.java b/test/dorkbox/util/StorageTest.java index 6744a9a..5f34c63 100644 --- a/test/dorkbox/util/StorageTest.java +++ b/test/dorkbox/util/StorageTest.java @@ -25,16 +25,9 @@ import org.junit.Before; import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; -import org.slf4j.Logger; - -import com.esotericsoftware.kryo.Kryo; -import com.esotericsoftware.kryo.Serializer; -import com.esotericsoftware.kryo.io.Input; -import com.esotericsoftware.kryo.io.Output; import dorkbox.util.storage.Storage; import dorkbox.util.storage.StorageSystem; -import io.netty.buffer.ByteBuf; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public @@ -48,70 +41,6 @@ class StorageTest { private static final File TEST_DB = new File("sampleFile.records"); - private static final SerializationManager manager = new SerializationManager() { - Kryo kryo = new Kryo(); - - @Override - public - void register(final Class clazz) { - kryo.register(clazz); - } - - @Override - public - void register(final Class clazz, final Serializer serializer) { - kryo.register(clazz, serializer); - } - - @Override - public - void register(final Class type, final Serializer serializer, final int id) { - kryo.register(type, serializer, id); - } - - @Override - public - void write(final ByteBuf buffer, final Object message) { - final Output output = new Output(); - writeFullClassAndObject(null, output, message); - buffer.writeBytes(output.getBuffer()); - } - - @Override - public - Object read(final ByteBuf buffer, final int length) throws IOException { - final Input input = new Input(); - buffer.readBytes(input.getBuffer()); - - final Object o = readFullClassAndObject(null, input); - buffer.skipBytes(input.position()); - - return o; - } - - @Override - public - void writeFullClassAndObject(final Logger logger, final Output output, final Object value) { - kryo.writeClassAndObject(output, value); - } - - @Override - public - Object readFullClassAndObject(final Logger logger, final Input input) throws IOException { - return kryo.readClassAndObject(input); - } - - @Override - public - void finishInit() { - } - - @Override - public - boolean initialized() { - return false; - } - }; static void log(String s) { @@ -137,8 +66,7 @@ class StorageTest { TEST_DB.delete(); Storage storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); int numberOfRecords1 = storage.size(); long size1 = storage.getFileSize(); @@ -150,8 +78,7 @@ class StorageTest { storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); int numberOfRecords2 = storage.size(); long size2 = storage.getFileSize(); @@ -169,8 +96,7 @@ class StorageTest { try { Storage storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); for (int i = 0; i < total; i++) { add(storage, i); @@ -179,8 +105,7 @@ class StorageTest { StorageSystem.close(storage); storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); for (int i = 0; i < total; i++) { String record1Data = createData(i); @@ -207,8 +132,7 @@ class StorageTest { try { Storage storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); for (int i = 0; i < total; i++) { log("adding record " + i + "..."); @@ -224,8 +148,7 @@ class StorageTest { storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); String dataCheck = createData(total - 1); log("reading record " + (total - 1) + "..."); @@ -254,8 +177,7 @@ class StorageTest { try { Storage storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); for (int i = 0; i < total; i++) { add(storage, i); @@ -277,8 +199,7 @@ class StorageTest { storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); for (int i = 0; i < total; i++) { String dataCheck = createData(i); String readRecord = readRecord(storage, i); @@ -299,8 +220,7 @@ class StorageTest { try { Storage storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); for (int i = 0; i < total; i++) { add(storage, i); @@ -317,8 +237,7 @@ class StorageTest { storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); for (int i = 0; i < total; i++) { String dataCheck = createData(i); @@ -340,8 +259,7 @@ class StorageTest { try { Storage storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); for (int i = 0; i < total; i++) { String addRecord = add(storage, i); @@ -353,8 +271,7 @@ class StorageTest { storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); for (int i = 0; i < total; i++) { String dataCheck = createData(i); @@ -377,8 +294,7 @@ class StorageTest { StorageSystem.close(storage); storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); data2 = storage.getAndPut(createKey, new Data()); Assert.assertEquals("Object is not the same", data, data2); @@ -401,8 +317,7 @@ class StorageTest { try { Storage storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); for (int i = 0; i < total; i++) { String addRecord = add(storage, i); @@ -414,8 +329,7 @@ class StorageTest { storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); for (int i = 0; i < total; i++) { String dataCheck = createData(i); @@ -447,8 +361,7 @@ class StorageTest { storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); // check 9 again readRecord = readRecord(storage, 9); @@ -472,8 +385,7 @@ class StorageTest { try { Storage storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); for (int i = 0; i < total; i++) { String addRecord = add(storage, i); @@ -485,8 +397,7 @@ class StorageTest { storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); String updateRecord = updateRecord(storage, 3, createData(3) + "new"); String readRecord = readRecord(storage, 3); @@ -495,8 +406,7 @@ class StorageTest { StorageSystem.close(storage); storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); readRecord = readRecord(storage, 3); Assert.assertEquals("Object is not the same", updateRecord, readRecord); @@ -506,8 +416,7 @@ class StorageTest { StorageSystem.close(storage); storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); readRecord = readRecord(storage, 3); Assert.assertEquals("Object is not the same", updateRecord, readRecord); @@ -515,8 +424,7 @@ class StorageTest { StorageSystem.close(storage); storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); updateRecord = updateRecord(storage, 0, createData(0) + "new"); readRecord = readRecord(storage, 0); @@ -536,8 +444,7 @@ class StorageTest { try { Storage storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); for (int i = 0; i < total; i++) { Data data = new Data(); @@ -553,8 +460,7 @@ class StorageTest { storage = StorageSystem.Disk() .file(TEST_DB) - .serializer(manager) - .make(); + .build(); for (int i = 0; i < total; i++) { String createKey = createKey(i);