Refactored Storage.make() -> Storage.build(). Removed IOExceptions when

data cannot be read (an error will be emitted via logger or StdErr, and
will return null). Storage now has a default (kryo based) serialization
manager if none is specified.
This commit is contained in:
nathan 2017-08-02 21:54:01 +02:00
parent 13697c3d61
commit 973d6d99c2
7 changed files with 189 additions and 170 deletions

View File

@ -0,0 +1,97 @@
/*
* Copyright 2014 dorkbox, llc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package dorkbox.util.storage;
import java.io.IOException;
import org.slf4j.Logger;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import com.esotericsoftware.minlog.Log;
import dorkbox.util.SerializationManager;
import io.netty.buffer.ByteBuf;
class DefaultStorageSerializationManager implements SerializationManager {
private Kryo kryo = new Kryo() {{
// we don't want logging from Kryo...
Log.set(Log.LEVEL_ERROR);
}};
@Override
public
void register(final Class<?> clazz) {
kryo.register(clazz);
}
@Override
public
void register(final Class<?> clazz, final Serializer<?> serializer) {
kryo.register(clazz, serializer);
}
@Override
public
void register(final Class<?> type, final Serializer<?> serializer, final int id) {
kryo.register(type, serializer, id);
}
@Override
public
void write(final ByteBuf buffer, final Object message) {
final Output output = new Output();
writeFullClassAndObject(null, output, message);
buffer.writeBytes(output.getBuffer());
}
@Override
public
Object read(final ByteBuf buffer, final int length) throws IOException {
final Input input = new Input();
buffer.readBytes(input.getBuffer());
final Object o = readFullClassAndObject(null, input);
buffer.skipBytes(input.position());
return o;
}
@Override
public
void writeFullClassAndObject(final Logger logger, final Output output, final Object value) {
kryo.writeClassAndObject(output, value);
}
@Override
public
Object readFullClassAndObject(final Logger logger, final Input input) throws IOException {
return kryo.readClassAndObject(input);
}
@Override
public
void finishInit() {
}
@Override
public
boolean initialized() {
return false;
}
}

View File

@ -122,45 +122,37 @@ class DiskStorage implements Storage {
/**
* Reads a object using the default (blank) key, and casts it to the expected class
*
* @throws IOException if there is a problem deserializing data from disk
*/
@Override
public final
<T> T get() throws IOException {
<T> T get() {
return get0(this.defaultKey);
}
/**
* Reads a object using the specific key, and casts it to the expected class
*
* @throws IOException if there is a problem deserializing data from disk
*/
@Override
public final
<T> T get(String key) throws IOException {
<T> T get(String key) {
return get0(new StorageKey(key));
}
/**
* Reads a object using the specific key, and casts it to the expected class
*
* @throws IOException if there is a problem deserializing data from disk
*/
@Override
public final
<T> T get(byte[] key) throws IOException {
<T> T get(byte[] key) {
return get0(new StorageKey(key));
}
/**
* Reads a object using the specific key, and casts it to the expected class
*
* @throws IOException if there is a problem deserializing data from disk
*/
@Override
public final
<T> T get(StorageKey key) throws IOException {
<T> T get(StorageKey key) {
return get0(key);
}
@ -173,7 +165,7 @@ class DiskStorage implements Storage {
*/
@Override
public
<T> T getAndPut(T data) throws IOException {
<T> T getAndPut(T data) {
return getAndPut(this.defaultKey, data);
}
@ -184,7 +176,7 @@ class DiskStorage implements Storage {
*/
@Override
public
<T> T getAndPut(String key, T data) throws IOException {
<T> T getAndPut(String key, T data) {
StorageKey wrap = new StorageKey(key);
return getAndPut(wrap, data);
@ -197,19 +189,21 @@ class DiskStorage implements Storage {
*/
@Override
public
<T> T getAndPut(byte[] key, T data) throws IOException {
<T> T getAndPut(byte[] key, T data) {
return getAndPut(new StorageKey(key), data);
}
/**
* Returns the saved data for the specified key. Also saves the data.
* Returns the saved data (or null) for the specified key. Also saves the data.
*
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
*
* @return NULL if the saved data was the wrong type for the specified key.
*/
@Override
@SuppressWarnings("unchecked")
public
<T> T getAndPut(StorageKey key, T data) throws IOException {
<T> T getAndPut(StorageKey key, T data) {
Object source = get0(key);
if (source == null) {
@ -222,7 +216,16 @@ class DiskStorage implements Storage {
final Class<?> savedCLass = source.getClass();
if (!expectedClass.isAssignableFrom(savedCLass)) {
throw new IOException("Saved value type '" + source.getClass() + "' is different that expected value");
String message = "Saved value type '" + savedCLass + "' is different than expected value '" + expectedClass + "'";
if (storage.logger != null) {
storage.logger.error(message);
}
else {
System.err.print(message);
}
return null;
}
}
@ -231,11 +234,9 @@ class DiskStorage implements Storage {
/**
* Reads a object from pending or from storage
*
* @throws IOException if there is a problem deserializing data from disk
*/
private
<T> T get0(StorageKey key) throws IOException {
<T> T get0(StorageKey key) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}

View File

@ -16,7 +16,6 @@
package dorkbox.util.storage;
import java.io.File;
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
/**
@ -99,7 +98,7 @@ class MemoryStorage implements Storage {
*/
@Override
public
<T> T getAndPut(T data) throws IOException {
<T> T getAndPut(T data) {
return getAndPut(this.defaultKey, data);
}
@ -110,7 +109,7 @@ class MemoryStorage implements Storage {
*/
@Override
public
<T> T getAndPut(String key, T data) throws IOException {
<T> T getAndPut(String key, T data) {
StorageKey wrap = new StorageKey(key);
return getAndPut(wrap, data);
@ -123,14 +122,14 @@ class MemoryStorage implements Storage {
*/
@Override
public
<T> T getAndPut(byte[] key, T data) throws IOException {
<T> T getAndPut(byte[] key, T data) {
return getAndPut(new StorageKey(key), data);
}
@SuppressWarnings("unchecked")
@Override
public
<T> T getAndPut(final StorageKey key, final T data) throws IOException {
<T> T getAndPut(final StorageKey key, final T data) {
final Object o = storage.get(key);
if (o == null) {
storage.put(key, data);

View File

@ -16,11 +16,11 @@
package dorkbox.util.storage;
import java.io.File;
import java.io.IOException;
/**
*
*/
@SuppressWarnings("unused")
public
interface Storage {
/**
@ -36,22 +36,22 @@ interface Storage {
/**
* Reads a object using the DEFAULT key ("") key, and casts it to the expected class
*/
<T> T get() throws IOException;
<T> T get();
/**
* Reads a object using the specific key, and casts it to the expected class.
*/
<T> T get(String key);
/**
* Reads a object using the specific key, and casts it to the expected class
*/
<T> T get(String key) throws IOException;
<T> T get(byte[] key);
/**
* Reads a object using the specific key, and casts it to the expected class
*/
<T> T get(byte[] key) throws IOException;
/**
* Reads a object using the specific key, and casts it to the expected class
*/
<T> T get(StorageKey key) throws IOException;
<T> T get(StorageKey key);
/**
* Uses the DEFAULT key ("") to return saved data.
@ -60,7 +60,7 @@ interface Storage {
*
* @param data This is the default value, and if there is no value with the key in the DB this default value will be saved.
*/
<T> T getAndPut(T data) throws IOException;
<T> T getAndPut(T data);
/**
* Returns the saved data for the specified key.
@ -68,7 +68,7 @@ interface Storage {
* @param key The key used to check if data already exists.
* @param data This is the default value, and if there is no value with the key in the DB this default value will be saved.
*/
<T> T getAndPut(String key, T data) throws IOException;
<T> T getAndPut(String key, T data);
/**
* Returns the saved data for the specified key.
@ -76,7 +76,7 @@ interface Storage {
* @param key The key used to check if data already exists.
* @param data This is the default value, and if there is no value with the key in the DB this default value will be saved.
*/
<T> T getAndPut(byte[] key, T data) throws IOException;
<T> T getAndPut(byte[] key, T data);
/**
* Returns the saved data for the specified key.
@ -84,7 +84,7 @@ interface Storage {
* @param key The key used to check if data already exists.
* @param data This is the default value, and if there is no value with the key in the DB this default value will be saved.
*/
<T> T getAndPut(StorageKey key, T data) throws IOException;
<T> T getAndPut(StorageKey key, T data);
/**
* Saves the given data to storage with the associated key.

View File

@ -33,7 +33,6 @@ import java.util.concurrent.locks.ReentrantLock;
import org.slf4j.Logger;
import com.esotericsoftware.kryo.KryoException;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
@ -46,8 +45,9 @@ import dorkbox.util.SerializationManager;
// Also, file locks on linux are ADVISORY. if an app doesn't care about locks, then it can do stuff -- even if locked by another app
@SuppressWarnings("unused")
class StorageBase {
private final Logger logger;
protected final Logger logger;
// File pointer to the data start pointer header.
@ -101,7 +101,7 @@ class StorageBase {
private final Input input;
// input/output write buffer size before flushing to/from the file
public static final int BUFFER_SIZE = 1024;
private static final int BUFFER_SIZE = 1024;
/**
@ -249,10 +249,10 @@ class StorageBase {
}
/**
* @return an object for a specified key form referenceCache FIRST, then from DISK
* @return an object for a specified key form referenceCache FIRST, then from DISK. NULL if it doesn't exist or there was an error.
*/
final
<T> T get(StorageKey key) throws IOException {
<T> T get(StorageKey key) {
// NOT protected by lock
Metadata meta = this.memoryIndex.get(key);
@ -299,11 +299,14 @@ class StorageBase {
return readRecordData;
} catch (Exception e) {
if (e instanceof KryoException && e.getMessage().contains("(missing no-arg constructor)")) {
throw new IOException("Cannot get data from disk: " + e.getMessage().substring(0, e.getMessage().indexOf(OS.LINE_SEPARATOR)));
} else {
throw new IOException("Cannot get data from disk", e);
String message = e.getMessage().substring(0,e.getMessage().indexOf(OS.LINE_SEPARATOR));
if (logger != null) {
logger.error("Error reading data from disk: {}", message);
}
else {
System.err.print("Error reading data from disk: " + message);
}
return null;
}
}

View File

@ -25,6 +25,7 @@ import org.slf4j.Logger;
import org.slf4j.helpers.NOPLogger;
import dorkbox.util.FileUtil;
import dorkbox.util.OS;
import dorkbox.util.SerializationManager;
public
@ -47,6 +48,7 @@ class StorageSystem {
.addShutdownHook(shutdownHook);
}
/**
* Creates a persistent, on-disk storage system. Writes to disk are queued, so it is recommended to NOT edit/change an object after
* it has been put into storage, or whenever it does changes, make sure to put it back into storage (to update the saved record)
@ -212,13 +214,13 @@ class StorageSystem {
* Makes the storage system
*/
public
Storage make() {
Storage build() {
if (this.file == null) {
throw new IllegalArgumentException("file cannot be null!");
}
if (this.serializationManager == null) {
throw new IllegalArgumentException("serializer cannot be null!");
this.serializationManager = createDefaultSerializationManager();
}
// if we load from a NEW storage at the same location as an ALREADY EXISTING storage,
@ -244,13 +246,24 @@ class StorageSystem {
storage = new DiskStorage(this.file, this.serializationManager, this.readOnly, this.logger);
storages.put(this.file, storage);
} catch (IOException e) {
logger.error("Unable to open storage", e);
String message = e.getMessage().substring(0,e.getMessage().indexOf(OS.LINE_SEPARATOR));
if (logger != null) {
logger.error("Unable to open storage file at {}. {}", this.file, message);
}
else {
System.err.print("Unable to open storage file at " + this.file + ". " + message);
}
}
}
return storage;
}
}
private
SerializationManager createDefaultSerializationManager() {
return new DefaultStorageSerializationManager();
}
}
@ -261,10 +274,10 @@ class StorageSystem {
class MemoryMaker {
/**
* Makes the storage system
* Builds the storage system
*/
public
MemoryStorage make() throws IOException {
MemoryStorage build() {
return new MemoryStorage();
}
}

View File

@ -25,16 +25,9 @@ import org.junit.Before;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runners.MethodSorters;
import org.slf4j.Logger;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import dorkbox.util.storage.Storage;
import dorkbox.util.storage.StorageSystem;
import io.netty.buffer.ByteBuf;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public
@ -48,70 +41,6 @@ class StorageTest {
private static final File TEST_DB = new File("sampleFile.records");
private static final SerializationManager manager = new SerializationManager() {
Kryo kryo = new Kryo();
@Override
public
void register(final Class<?> clazz) {
kryo.register(clazz);
}
@Override
public
void register(final Class<?> clazz, final Serializer<?> serializer) {
kryo.register(clazz, serializer);
}
@Override
public
void register(final Class<?> type, final Serializer<?> serializer, final int id) {
kryo.register(type, serializer, id);
}
@Override
public
void write(final ByteBuf buffer, final Object message) {
final Output output = new Output();
writeFullClassAndObject(null, output, message);
buffer.writeBytes(output.getBuffer());
}
@Override
public
Object read(final ByteBuf buffer, final int length) throws IOException {
final Input input = new Input();
buffer.readBytes(input.getBuffer());
final Object o = readFullClassAndObject(null, input);
buffer.skipBytes(input.position());
return o;
}
@Override
public
void writeFullClassAndObject(final Logger logger, final Output output, final Object value) {
kryo.writeClassAndObject(output, value);
}
@Override
public
Object readFullClassAndObject(final Logger logger, final Input input) throws IOException {
return kryo.readClassAndObject(input);
}
@Override
public
void finishInit() {
}
@Override
public
boolean initialized() {
return false;
}
};
static
void log(String s) {
@ -137,8 +66,7 @@ class StorageTest {
TEST_DB.delete();
Storage storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
int numberOfRecords1 = storage.size();
long size1 = storage.getFileSize();
@ -150,8 +78,7 @@ class StorageTest {
storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
int numberOfRecords2 = storage.size();
long size2 = storage.getFileSize();
@ -169,8 +96,7 @@ class StorageTest {
try {
Storage storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
for (int i = 0; i < total; i++) {
add(storage, i);
@ -179,8 +105,7 @@ class StorageTest {
StorageSystem.close(storage);
storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
for (int i = 0; i < total; i++) {
String record1Data = createData(i);
@ -207,8 +132,7 @@ class StorageTest {
try {
Storage storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
for (int i = 0; i < total; i++) {
log("adding record " + i + "...");
@ -224,8 +148,7 @@ class StorageTest {
storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
String dataCheck = createData(total - 1);
log("reading record " + (total - 1) + "...");
@ -254,8 +177,7 @@ class StorageTest {
try {
Storage storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
for (int i = 0; i < total; i++) {
add(storage, i);
@ -277,8 +199,7 @@ class StorageTest {
storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
for (int i = 0; i < total; i++) {
String dataCheck = createData(i);
String readRecord = readRecord(storage, i);
@ -299,8 +220,7 @@ class StorageTest {
try {
Storage storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
for (int i = 0; i < total; i++) {
add(storage, i);
@ -317,8 +237,7 @@ class StorageTest {
storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
for (int i = 0; i < total; i++) {
String dataCheck = createData(i);
@ -340,8 +259,7 @@ class StorageTest {
try {
Storage storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
for (int i = 0; i < total; i++) {
String addRecord = add(storage, i);
@ -353,8 +271,7 @@ class StorageTest {
storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
for (int i = 0; i < total; i++) {
String dataCheck = createData(i);
@ -377,8 +294,7 @@ class StorageTest {
StorageSystem.close(storage);
storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
data2 = storage.getAndPut(createKey, new Data());
Assert.assertEquals("Object is not the same", data, data2);
@ -401,8 +317,7 @@ class StorageTest {
try {
Storage storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
for (int i = 0; i < total; i++) {
String addRecord = add(storage, i);
@ -414,8 +329,7 @@ class StorageTest {
storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
for (int i = 0; i < total; i++) {
String dataCheck = createData(i);
@ -447,8 +361,7 @@ class StorageTest {
storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
// check 9 again
readRecord = readRecord(storage, 9);
@ -472,8 +385,7 @@ class StorageTest {
try {
Storage storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
for (int i = 0; i < total; i++) {
String addRecord = add(storage, i);
@ -485,8 +397,7 @@ class StorageTest {
storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
String updateRecord = updateRecord(storage, 3, createData(3) + "new");
String readRecord = readRecord(storage, 3);
@ -495,8 +406,7 @@ class StorageTest {
StorageSystem.close(storage);
storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
readRecord = readRecord(storage, 3);
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
@ -506,8 +416,7 @@ class StorageTest {
StorageSystem.close(storage);
storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
readRecord = readRecord(storage, 3);
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
@ -515,8 +424,7 @@ class StorageTest {
StorageSystem.close(storage);
storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
updateRecord = updateRecord(storage, 0, createData(0) + "new");
readRecord = readRecord(storage, 0);
@ -536,8 +444,7 @@ class StorageTest {
try {
Storage storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
for (int i = 0; i < total; i++) {
Data data = new Data();
@ -553,8 +460,7 @@ class StorageTest {
storage = StorageSystem.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
.build();
for (int i = 0; i < total; i++) {
String createKey = createKey(i);