dorkbox/Utilities
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

wip - creating disk or memory based storage

Browse Source
This commit is contained in:
nathan 2015-07-07 01:26:17 +02:00
parent 927a5aece9
commit 08f7872a40
8 changed files with 1603 additions and 847 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

516
Dorkbox-Util/src/dorkbox/util/storage/DiskStorage.java Normal file
View File

@ -0,0 +1,516 @@
/*
* Copyright 2014 dorkbox, llc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package dorkbox.util.storage;
import dorkbox.util.DelayTimer;
import dorkbox.util.SerializationManager;
import dorkbox.util.bytes.ByteArrayWrapper;
import java.io.File;
import java.io.IOException;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
/**
* Nothing spectacular about this storage -- it allows for persistent storage of objects to disk.
* <p/>
* Be wary of opening the database file in different instances. Even with file-locks, you can corrupt the data.
*/
@SuppressWarnings("unused")
public
class DiskStorage implements DiskStorageIfface {
private final DelayTimer timer;
private final ByteArrayWrapper defaultKey;
private final StorageBase storage;
private final AtomicInteger references = new AtomicInteger(1);
private final ReentrantLock actionLock = new ReentrantLock();
private final AtomicBoolean isOpen = new AtomicBoolean(false);
private volatile long milliSeconds = 3000L;
private volatile Map<ByteArrayWrapper, Object> actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
/**
* Creates or opens a new database file.
*/
protected
DiskStorage(File storageFile, SerializationManager serializationManager) throws IOException {
this.storage = new StorageBase(storageFile, serializationManager);
this.defaultKey = ByteArrayWrapper.wrap("");
this.timer = new DelayTimer("Storage Writer", false, new DelayTimer.Callback() {
@Override
public
void execute() {
Map<ByteArrayWrapper, Object> actions = DiskStorage.this.actionMap;
ReentrantLock actionLock2 = DiskStorage.this.actionLock;
try {
actionLock2.lock();
// do a fast swap on the actionMap.
DiskStorage.this.actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
} finally {
actionLock2.unlock();
}
DiskStorage.this.storage.doActionThings(actions);
}
});
this.isOpen.set(true);
}
/**
* Returns the number of objects in the database.
* <p/>
* SLOW because this must save all data to disk first!
*/
@Override
public final
int size() {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
// flush actions
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
return this.storage.size();
}
/**
* Checks if there is a object corresponding to the given key.
*/
@Override
public final
boolean contains(String key) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
final ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
// check if our pending actions has it, or if our storage index has it
return this.actionMap.containsKey(wrap) || this.storage.contains(wrap);
}
/**
* Reads a object using the default (blank) key, and casts it to the expected class
*/
@Override
public final
<T> T get() {
return get0(this.defaultKey);
}
/**
* Reads a object using the specific key, and casts it to the expected class
*/
@Override
public final
<T> T get(String key) {
return get0(ByteArrayWrapper.wrap(key));
}
/**
* Reads a object using the specific key, and casts it to the expected class
*/
@Override
public final
<T> T get(byte[] key) {
return get0(ByteArrayWrapper.wrap(key));
}
/**
* Reads a object using the specific key, and casts it to the expected class
*/
@Override
public final
<T> T get(ByteArrayWrapper key) {
return get0(key);
}
/**
* Uses the DEFAULT key ("") to return saved data.
* <p/>
* This will check to see if there is an associated key for that data, if not - it will use data as the default
*
* @param data The data that will hold the copy of the data from disk
*/
@Override
public
<T> T load(T data) throws IOException {
return load(this.defaultKey, data);
}
/**
* Returns the saved data for the specified key.
*
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
*/
@Override
public
<T> T load(String key, T data) throws IOException {
ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
return load(wrap, data);
}
/**
* Returns the saved data for the specified key.
*
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
*/
@Override
public
<T> T load(byte[] key, T data) throws IOException {
return load(ByteArrayWrapper.wrap(key), data);
}
/**
* Returns the saved data for the specified key.
*
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
*/
@Override
@SuppressWarnings("unchecked")
public
<T> T load(ByteArrayWrapper key, T data) throws IOException {
Object source = get0(key);
if (source == null) {
// returned was null, so we should take value as the default
put(key, data);
return data;
}
else {
final Class<?> expectedClass = data.getClass();
final Class<?> savedCLass = source.getClass();
if (!expectedClass.isAssignableFrom(savedCLass)) {
throw new IOException("Saved value type '" + source.getClass() + "' is different that expected value");
}
}
return (T) source;
}
/**
* Reads a object from pending or from storage
*/
private
<T> T get0(ByteArrayWrapper key) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
// if the object in is pending, we get it from there
try {
this.actionLock.lock();
Object object = this.actionMap.get(key);
if (object != null) {
@SuppressWarnings("unchecked")
T returnObject = (T) object;
return returnObject;
}
} finally {
this.actionLock.unlock();
}
// not found, so we have to go find it on disk
return this.storage.get(key);
}
/**
* Saves the given data to storage with the associated key.
* <p/>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
@Override
public final
void put(String key, Object object) {
put(ByteArrayWrapper.wrap(key), object);
}
/**
* Saves the given data to storage with the associated key.
* <p/>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
@Override
public final
void put(byte[] key, Object object) {
put(ByteArrayWrapper.wrap(key), object);
}
/**
* Saves the given data to storage with the associated key.
* <p/>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
@Override
public final
void put(ByteArrayWrapper key, Object object) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
action(key, object);
// timer action runs on TIMER thread, not this thread
this.timer.delay(this.milliSeconds);
}
/**
* Adds the given object to the storage using a default (blank) key, OR -- if it has been registered, using it's registered key
* <p/>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
@Override
public final
void put(Object object) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
action(this.defaultKey, object);
// timer action runs on TIMER thread, not this thread
this.timer.delay(this.milliSeconds);
}
/**
* Deletes an object from storage. To ALSO remove from the cache, use unRegister(key)
*
* @return true if the delete was successful. False if there were problems deleting the data.
*/
@Override
public final
boolean delete(String key) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
return this.storage.delete(wrap);
}
/**
* Closes the database and file.
*/
void close() {
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
// have to "close" it after we run the timer!
this.isOpen.set(false);
this.storage.close();
}
/**
* @return the file that backs this storage
*/
@Override
public final
File getFile() {
return this.storage.getFile();
}
/**
* Gets the backing file size.
*
* @return -1 if there was an error
*/
@Override
public final
long getFileSize() {
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
return this.storage.getFileSize();
}
/**
* @return true if there are objects queued to be written?
*/
@Override
public final
boolean hasWriteWaiting() {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
return this.timer.isWaiting();
}
/**
* @return the delay in milliseconds this will wait after the last action to flush the data to the disk
*/
@Override
public final
long getSaveDelay() {
return this.milliSeconds;
}
/**
* @param milliSeconds milliseconds to wait
*/
@Override
public final
void setSaveDelay(long milliSeconds) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
this.milliSeconds = milliSeconds;
}
/**
* @return the version of data stored in the database
*/
@Override
public final
int getVersion() {
return this.storage.getVersion();
}
/**
* Sets the version of data stored in the database
*/
@Override
public final
void setVersion(int version) {
this.storage.setVersion(version);
}
private
void action(ByteArrayWrapper key, Object object) {
try {
this.actionLock.lock();
// push action to map
this.actionMap.put(key, object);
} finally {
this.actionLock.unlock();
}
}
void increaseReference() {
this.references.incrementAndGet();
}
/**
* return true when this is the last reference
*/
boolean decrementReference() {
return this.references.decrementAndGet() <= 0;
}
@Override
protected
Object clone() throws CloneNotSupportedException {
return super.clone();
}
/**
* Save the storage to disk, immediately.
* <p/>
* This will save the ALL of the pending save actions to the file
*/
@Override
public final
void commit() {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
}
/**
* Save the storage to disk, immediately.
* <p/>
* This will save the ALL of the pending save actions to the file
*/
@Override
public
void commit(final String key, final Object object) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
action(ByteArrayWrapper.wrap(key), object);
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
}
/**
* Save the storage to disk, immediately.
* <p/>
* This will save the ALL of the pending save actions to the file
*/
@Override
public
void commit(final byte[] key, final Object object) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
action(ByteArrayWrapper.wrap(key), object);
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
}
/**
* Save the storage to disk, immediately.
* <p/>
* This will save the ALL of the pending save actions to the file
*/
@Override
public
void commit(final ByteArrayWrapper key, final Object object) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
action(key, object);
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
}
}

177
Dorkbox-Util/src/dorkbox/util/storage/DiskStorageIfface.java Normal file
View File

@ -0,0 +1,177 @@
package dorkbox.util.storage;
import dorkbox.util.bytes.ByteArrayWrapper;
import java.io.File;
import java.io.IOException;
/**
*
*/
public
interface DiskStorageIfface {
/**
* Returns the number of objects in the database.
*/
int size();
/**
* Checks if there is a object corresponding to the given key.
*/
boolean contains(String key);
/**
* Reads a object using the default (blank) key, and casts it to the expected class
*/
<T> T get();
/**
* Reads a object using the specific key, and casts it to the expected class
*/
<T> T get(String key);
/**
* Reads a object using the specific key, and casts it to the expected class
*/
<T> T get(byte[] key);
/**
* Reads a object using the specific key, and casts it to the expected class
*/
<T> T get(ByteArrayWrapper key);
/**
* Uses the DEFAULT key ("") to return saved data.
* <p/>
* This will check to see if there is an associated key for that data, if not - it will use data as the default
*
* @param data The data that will hold the copy of the data from disk
*/
<T> T load(T data) throws IOException;
/**
* Returns the saved data for the specified key.
*
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
*/
<T> T load(String key, T data) throws IOException;
/**
* Returns the saved data for the specified key.
*
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
*/
<T> T load(byte[] key, T data) throws IOException;
/**
* Returns the saved data for the specified key.
*
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
*/
@SuppressWarnings("unchecked")
<T> T load(ByteArrayWrapper key, T data) throws IOException;
/**
* Saves the given data to storage with the associated key.
* <p/>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
void put(String key, Object data);
/**
* Saves the given data to storage with the associated key.
* <p/>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
void put(byte[] key, Object data);
/**
* Saves the given data to storage with the associated key.
* <p/>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
void put(ByteArrayWrapper key, Object data);
/**
* Adds the given object to the storage using a default (blank) key, OR -- if it has been registered, using it's registered key
* <p/>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
void put(Object data);
/**
* Deletes an object from storage. To ALSO remove from the cache, use unRegister(key)
*
* @return true if the delete was successful. False if there were problems deleting the data.
*/
boolean delete(String key);
/**
* @return the file that backs this storage
*/
File getFile();
/**
* Gets the backing file size.
*
* @return -1 if there was an error
*/
long getFileSize();
/**
* @return true if there are objects queued to be written?
*/
boolean hasWriteWaiting();
/**
* @return the delay in milliseconds this will wait after the last action to flush the data to the disk
*/
long getSaveDelay();
/**
* @param milliSeconds milliseconds to wait
*/
void setSaveDelay(long milliSeconds);
/**
* @return the version of data stored in the database
*/
int getVersion();
/**
* Sets the version of data stored in the database
*/
void setVersion(int version);
/**
* Save the storage to disk, immediately.
* <p/>
* This will save the ALL of the pending save actions to the file
*/
void commit();
/**
* Save the storage to disk, immediately.
* <p/>
* This will save the ALL of the pending save actions to the file
*/
void commit(String key, Object object);
/**
* Save the storage to disk, immediately.
* <p/>
* This will save the ALL of the pending save actions to the file
*/
void commit(byte[] key, Object object);
/**
* Save the storage to disk, immediately.
* <p/>
* This will save the ALL of the pending save actions to the file
*/
void commit(ByteArrayWrapper key, Object object);
}

176
Dorkbox-Util/src/dorkbox/util/storage/MakeStorage.java Normal file
View File

@ -0,0 +1,176 @@
package dorkbox.util.storage;
import dorkbox.util.SerializationManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
public
class MakeStorage {
private static final Logger logger = LoggerFactory.getLogger(DiskStorage.class);
@SuppressWarnings("SpellCheckingInspection")
private static final Map<File, DiskStorageIfface> storages = new HashMap<File, DiskStorageIfface>(1);
public static
DiskMaker Disk() {
return new DiskMaker();
}
public static
MemoryMaker Memory() {
return new MemoryMaker();
}
/**
* Closes the storage.
*/
public static
void close(File file) {
synchronized (storages) {
DiskStorageIfface storage = storages.get(file);
if (storage != null) {
if (storage instanceof DiskStorage) {
final DiskStorage diskStorage = (DiskStorage) storage;
boolean isLastOne = diskStorage.decrementReference();
if (isLastOne) {
diskStorage.close();
storages.remove(file);
}
}
}
}
}
/**
* Closes the storage.
*/
public static
void close(DiskStorageIfface _storage) {
synchronized (storages) {
File file = _storage.getFile();
DiskStorageIfface storage = storages.get(file);
if (storage != null) {
if (storage instanceof DiskStorage) {
final DiskStorage diskStorage = (DiskStorage) storage;
boolean isLastOne = diskStorage.decrementReference();
if (isLastOne) {
diskStorage.close();
storages.remove(file);
}
}
}
}
}
public static
void shutdown() {
synchronized (storages) {
Collection<DiskStorageIfface> values = storages.values();
for (DiskStorageIfface storage : values) {
if (storage instanceof DiskStorage) {
//noinspection StatementWithEmptyBody
final DiskStorage diskStorage = (DiskStorage) storage;
while (!diskStorage.decrementReference()) {
}
diskStorage.close();
}
}
storages.clear();
}
}
public static
void delete(File file) {
synchronized (storages) {
DiskStorageIfface remove = storages.remove(file);
if (remove != null && remove instanceof DiskStorage) {
((DiskStorage) remove).close();
}
//noinspection ResultOfMethodCallIgnored
file.delete();
}
}
public static
void delete(DiskStorageIfface storage) {
File file = storage.getFile();
delete(file);
}
public static
class DiskMaker {
private File file;
private SerializationManager serializationManager;
public
DiskMaker file(File file) {
this.file = file;
return this;
}
public
DiskMaker serializer(SerializationManager serializationManager) {
this.serializationManager = serializationManager;
return this;
}
public
DiskStorageIfface make() {
if (file == null) {
throw new IllegalArgumentException("file cannot be null!");
}
// if we load from a NEW storage at the same location as an ALREADY EXISTING storage,
// without saving the existing storage first --- whoops!
synchronized (storages) {
DiskStorageIfface storage = storages.get(file);
if (storage != null) {
if (storage instanceof DiskStorage) {
boolean waiting = storage.hasWriteWaiting();
// we want this storage to be in a fresh state
if (waiting) {
storage.commit();
}
((DiskStorage) storage).increaseReference();
}
else {
throw new RuntimeException("Unable to change storage types for: " + file);
}
}
else {
try {
storage = new DiskStorage(file, serializationManager);
storages.put(file, storage);
} catch (IOException e) {
logger.error("Unable to open storage", e);
}
}
return storage;
}
}
}
public static
class MemoryMaker {
private SerializationManager serializationManager;
public
MemoryMaker serializer(SerializationManager serializationManager) {
this.serializationManager = serializationManager;
return this;
}
MemoryStorage make() {
return null;
}
}
}

253
Dorkbox-Util/src/dorkbox/util/storage/MemoryStorage.java Normal file
View File

@ -0,0 +1,253 @@
package dorkbox.util.storage;
import dorkbox.util.bytes.ByteArrayWrapper;
import java.io.File;
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
/**
*
*/
public
class MemoryStorage implements DiskStorageIfface {
private final ConcurrentHashMap<ByteArrayWrapper, Object> storage;
private final ByteArrayWrapper defaultKey;
private int version;
private
MemoryStorage() throws IOException {
this.storage = new ConcurrentHashMap<ByteArrayWrapper, Object>();
this.defaultKey = ByteArrayWrapper.wrap("");
}
/**
* Returns the number of objects in the database.
*/
@Override
public
int size() {
return storage.size();
}
/**
* Checks if there is a object corresponding to the given key.
*/
@Override
public
boolean contains(final String key) {
return storage.containsKey(ByteArrayWrapper.wrap(key));
}
/**
* Reads a object using the default (blank) key, and casts it to the expected class
*/
@SuppressWarnings("unchecked")
@Override
public
<T> T get() {
return (T) storage.get(defaultKey);
}
/**
* Reads a object using the specific key, and casts it to the expected class
*/
@Override
public
<T> T get(final String key) {
return get(ByteArrayWrapper.wrap(key));
}
/**
* Reads a object using the specific key, and casts it to the expected class
*/
@Override
public
<T> T get(final byte[] key) {
return get(ByteArrayWrapper.wrap(key));
}
/**
* Reads a object using the specific key, and casts it to the expected class
*/
@SuppressWarnings("unchecked")
@Override
public
<T> T get(final ByteArrayWrapper key) {
return (T) storage.get(key);
}
/**
* Uses the DEFAULT key ("") to return saved data.
* <p/>
* This will check to see if there is an associated key for that data, if not - it will use data as the default
*
* @param data The data that will hold the copy of the data from disk
*/
@Override
public
<T> T load(T data) throws IOException {
return load(this.defaultKey, data);
}
/**
* Returns the saved data for the specified key.
*
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
*/
@Override
public
<T> T load(String key, T data) throws IOException {
ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
return load(wrap, data);
}
/**
* Returns the saved data for the specified key.
*
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
*/
@Override
public
<T> T load(byte[] key, T data) throws IOException {
return load(ByteArrayWrapper.wrap(key), data);
}
@SuppressWarnings("unchecked")
@Override
public
<T> T load(final ByteArrayWrapper key, final T data) throws IOException {
final Object o = storage.get(key);
if (o == null) {
storage.put(key, data);
}
return (T) o;
}
/**
* Saves the given data to storage with the associated key.
* <p/>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
@Override
public
void put(final String key, final Object data) {
put(ByteArrayWrapper.wrap(key), data);
}
/**
* Saves the given data to storage with the associated key.
* <p/>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
@Override
public
void put(final byte[] key, final Object data) {
put(ByteArrayWrapper.wrap(key), data);
}
/**
* Saves the given data to storage with the associated key.
* <p/>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
@Override
public
void put(final ByteArrayWrapper key, final Object object) {
storage.put(key, object);
}
/**
* Saves the given data to storage with the associated key.
* <p/>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
@Override
public
void put(final Object data) {
put(defaultKey, data);
}
/**
* Deletes an object from storage. To ALSO remove from the cache, use unRegister(key)
*
* @return true if the delete was successful. False if there were problems deleting the data.
*/
@Override
public
boolean delete(final String key) {
storage.remove(ByteArrayWrapper.wrap(key));
return true;
}
@Override
public
File getFile() {
return null;
}
@Override
public
long getFileSize() {
return 0;
}
@Override
public
boolean hasWriteWaiting() {
return false;
}
@Override
public
long getSaveDelay() {
return 0;
}
@Override
public
void setSaveDelay(final long milliSeconds) {
}
@Override
public synchronized
int getVersion() {
return version;
}
@Override
public synchronized
void setVersion(final int version) {
this.version = version;
}
@Override
public
void commit() {
// no-op
}
@Override
public
void commit(final String key, final Object object) {
// no-op
}
@Override
public
void commit(final byte[] key, final Object object) {
// no-op
}
@Override
public
void commit(final ByteArrayWrapper key, final Object object) {
// no-op
}
}

97
Dorkbox-Util/src/dorkbox/util/storage/Metadata.java
View File

@ -15,6 +15,11 @@
*/
package dorkbox.util.storage;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import dorkbox.util.SerializationManager;
import dorkbox.util.bytes.ByteArrayWrapper;
import java.io.ByteArrayOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
@ -24,22 +29,18 @@ import java.nio.channels.FileLock;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import dorkbox.util.bytes.ByteArrayWrapper;
public class Metadata {
@SuppressWarnings("unused")
public
class Metadata {
// The length of a key in the index.
// SHA256 is 32 bytes long.
private static final int KEY_SIZE = 32;
private static final int KEY_SIZE = 32;
// Number of bytes in the record header.
private static final int POINTER_INFO_SIZE = 16;
// The total length of one index entry - the key length plus the record header length.
static final int INDEX_ENTRY_LENGTH = KEY_SIZE + POINTER_INFO_SIZE;
static final int INDEX_ENTRY_LENGTH = KEY_SIZE + POINTER_INFO_SIZE;
/**
@ -60,12 +61,12 @@ public class Metadata {
/**
* Actual number of bytes of data held in this record (4 bytes).
*/
volatile int dataCount;
volatile int dataCount;
/**
* Number of bytes of data that this record can hold (4 bytes).
*/
volatile int dataCapacity;
volatile int dataCapacity;
/**
@ -78,7 +79,8 @@ public class Metadata {
/**
* Returns a file pointer in the index pointing to the first byte in the KEY located at the given index position.
*/
static final long getMetaDataPointer(int position) {
static
long getMetaDataPointer(int position) {
return StorageBase.FILE_HEADERS_REGION_LENGTH + INDEX_ENTRY_LENGTH * position;
}
@ -86,17 +88,20 @@ public class Metadata {
* Returns a file pointer in the index pointing to the first byte in the RECORD pointer located at the given index
* position.
*/
static final long getDataPointer(int position) {
long l = Metadata.getMetaDataPointer(position) + KEY_SIZE;
return l;
static
long getDataPointer(int position) {
return Metadata.getMetaDataPointer(position) + KEY_SIZE;
}
private Metadata(ByteArrayWrapper key) {
private
Metadata(ByteArrayWrapper key) {
this.key = key;
}
/** we don't know how much data there is until AFTER we write the data */
/**
* we don't know how much data there is until AFTER we write the data
*/
Metadata(ByteArrayWrapper key, int recordIndex, long dataPointer) {
this(key, recordIndex, dataPointer, 0);
}
@ -123,11 +128,13 @@ public class Metadata {
/**
* Reads the ith HEADER (key + metadata) from the index.
*/
static Metadata readHeader(RandomAccessFile file, int position) throws IOException {
static
Metadata readHeader(RandomAccessFile file, int position) throws IOException {
byte[] buf = new byte[KEY_SIZE];
long origHeaderKeyPointer = Metadata.getMetaDataPointer(position);
FileLock lock = file.getChannel().lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true);
FileLock lock = file.getChannel()
.lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true);
file.seek(origHeaderKeyPointer);
file.readFully(buf);
lock.release();
@ -136,7 +143,8 @@ public class Metadata {
r.indexPosition = position;
long recordHeaderPointer = Metadata.getDataPointer(position);
lock = file.getChannel().lock(origHeaderKeyPointer, KEY_SIZE, true);
lock = file.getChannel()
.lock(origHeaderKeyPointer, KEY_SIZE, true);
file.seek(recordHeaderPointer);
r.dataPointer = file.readLong();
r.dataCapacity = file.readInt();
@ -153,7 +161,8 @@ public class Metadata {
void writeMetaDataInfo(RandomAccessFile file) throws IOException {
long recordKeyPointer = Metadata.getMetaDataPointer(this.indexPosition);
FileLock lock = file.getChannel().lock(recordKeyPointer, KEY_SIZE, false);
FileLock lock = file.getChannel()
.lock(recordKeyPointer, KEY_SIZE, false);
file.seek(recordKeyPointer);
file.write(this.key.getBytes());
lock.release();
@ -162,7 +171,8 @@ public class Metadata {
void writeDataInfo(RandomAccessFile file) throws IOException {
long recordHeaderPointer = getDataPointer(this.indexPosition);
FileLock lock = file.getChannel().lock(recordHeaderPointer, POINTER_INFO_SIZE, false);
FileLock lock = file.getChannel()
.lock(recordHeaderPointer, POINTER_INFO_SIZE, false);
file.seek(recordHeaderPointer);
file.writeLong(this.dataPointer);
file.writeInt(this.dataCapacity);
@ -177,13 +187,17 @@ public class Metadata {
byte[] buf = new byte[KEY_SIZE];
long origHeaderKeyPointer = Metadata.getMetaDataPointer(this.indexPosition);
FileLock lock = file.getChannel().lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true);
FileLock lock = file.getChannel()
.lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true);
file.seek(origHeaderKeyPointer);
file.readFully(buf);
lock.release();
long newHeaderKeyPointer = Metadata.getMetaDataPointer(newIndex);
lock = file.getChannel().lock(newHeaderKeyPointer, INDEX_ENTRY_LENGTH, false);
lock = file.getChannel()
.lock(newHeaderKeyPointer, INDEX_ENTRY_LENGTH, false);
file.seek(newHeaderKeyPointer);
file.write(buf);
lock.release();
@ -205,7 +219,8 @@ public class Metadata {
this.dataPointer = position;
this.dataCapacity = this.dataCount;
FileLock lock = file.getChannel().lock(position, this.dataCount, false);
FileLock lock = file.getChannel()
.lock(position, this.dataCount, false);
// update the file size
file.setLength(position + this.dataCount);
@ -230,7 +245,8 @@ public class Metadata {
byte[] buf = new byte[this.dataCount];
// System.err.print("Reading data: " + this.indexPosition + " @ " + this.dataPointer + "-" + (this.dataPointer+this.dataCount) + " -- ");
FileLock lock = file.getChannel().lock(this.dataPointer, this.dataCount, true);
FileLock lock = file.getChannel()
.lock(this.dataPointer, this.dataCount, true);
file.seek(this.dataPointer);
file.readFully(buf);
lock.release();
@ -239,24 +255,29 @@ public class Metadata {
return buf;
}
/**
* Reads the record data for the given record header.
*/
/**
* Reads the record data for the given record header.
*/
<T> T readData(Kryo kryo, InflaterInputStream inputStream) {
static
<T> T readData(SerializationManager serializationManager, InflaterInputStream inputStream) {
Input input = new Input(inputStream, 1024); // read 1024 at a time
@SuppressWarnings("unchecked")
T readObject = (T) kryo.readClassAndObject(input);
T readObject = (T) serializationManager.readClassAndObject(input);
return readObject;
}
/**
* Writes data to the end of the file (which is where the datapointer is at)
*/
void writeDataFast(Kryo kryo, Object data, DeflaterOutputStream outputStream) throws IOException {
static
void writeDataFast(final SerializationManager serializationManager,
final Object data,
final RandomAccessFile file,
final DeflaterOutputStream outputStream) throws IOException {
// HAVE TO LOCK BEFORE THIS IS CALLED! (AND FREE AFTERWARDS!)
Output output = new Output(outputStream, 1024); // write 1024 at a time
kryo.writeClassAndObject(output, data);
serializationManager.writeClassAndObject(output, data);
output.flush();
outputStream.flush(); // sync-flush is enabled, so the output stream will finish compressing data.
@ -265,7 +286,8 @@ public class Metadata {
void writeData(ByteArrayOutputStream byteArrayOutputStream, RandomAccessFile file) throws IOException {
this.dataCount = byteArrayOutputStream.size();
FileLock lock = file.getChannel().lock(this.dataPointer, this.dataCount, false);
FileLock lock = file.getChannel()
.lock(this.dataPointer, this.dataCount, false);
FileOutputStream out = new FileOutputStream(file.getFD());
file.seek(this.dataPointer);
@ -275,8 +297,9 @@ public class Metadata {
}
@Override
public String toString() {
return "RecordHeader [dataPointer=" + this.dataPointer + ", dataCount=" + this.dataCount + ", dataCapacity="
+ this.dataCapacity + ", indexPosition=" + this.indexPosition + "]";
public
String toString() {
return "RecordHeader [dataPointer=" + this.dataPointer + ", dataCount=" + this.dataCount + ", dataCapacity=" + this.dataCapacity +
", indexPosition=" + this.indexPosition + "]";
}
}

568
Dorkbox-Util/src/dorkbox/util/storage/Storage.java
View File

@ -1,568 +0,0 @@
/*
* Copyright 2014 dorkbox, llc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package dorkbox.util.storage;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import org.bouncycastle.crypto.digests.SHA256Digest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import dorkbox.util.DelayTimer;
import dorkbox.util.OS;
import dorkbox.util.bytes.ByteArrayWrapper;
/**
* Nothing spectacular about this storage -- it allows for persistent storage of objects to disk.
*/
public class Storage {
private static final Logger logger = LoggerFactory.getLogger(Storage.class);
private static Map<File, Storage> storages = new HashMap<File, Storage>(1);
public static Storage open(String file) {
if (file == null || file.isEmpty()) {
throw new IllegalArgumentException("file cannot be null or empty!");
}
return open(new File(file));
}
/**
* Two types of storage.
* Raw) save/load a single object to disk (better for really large files)
* Normal) save/load key/value objects to disk (better for multiple types of data in a single file)
*/
public static Storage open(File file) {
if (file == null) {
throw new IllegalArgumentException("file cannot be null!");
}
// if we load from a NEW storage at the same location as an ALREADY EXISTING storage,
// without saving the existing storage first --- whoops!
synchronized (storages) {
Storage storage = storages.get(file);
if (storage != null) {
boolean waiting = storage.hasWriteWaiting();
// we want this storage to be in a fresh state
if (waiting) {
storage.saveNow();
}
storage.increaseReference();
} else {
try {
storage = new Storage(file);
storages.put(file, storage);
} catch (IOException e) {
logger.error("Unable to open storage", e);
}
}
return storage;
}
}
/**
* Closes the storage.
*/
public static void close(File file) {
synchronized (storages) {
Storage storage = storages.get(file);
if (storage != null) {
boolean isLastOne = storage.decrementReference();
if (isLastOne) {
storage.close();
storages.remove(file);
}
}
}
}
/**
* Closes the storage.
*/
public static void close(Storage _storage) {
synchronized (storages) {
File file = _storage.getFile();
Storage storage = storages.get(file);
if (storage != null) {
boolean isLastOne = storage.decrementReference();
if (isLastOne) {
storage.close();
storages.remove(file);
}
}
}
}
public static void shutdown() {
synchronized(storages) {
Collection<Storage> values = storages.values();
for (Storage storage : values) {
while (!storage.decrementReference()) {
}
storage.close();
}
storages.clear();
}
}
public static void delete(File file) {
synchronized(storages) {
Storage remove = storages.remove(file);
if (remove != null) {
remove.close();
}
file.delete();
}
}
public static void delete(Storage storage) {
File file = storage.getFile();
delete(file);
}
private static void copyFields(Object source, Object dest) {
Class<? extends Object> sourceClass = source.getClass();
Class<? extends Object> destClass = dest.getClass();
if (sourceClass != destClass) {
throw new IllegalArgumentException("Source and Dest objects are not of the same class!");
}
// have to walk up the object hierarchy.
while (destClass != Object.class) {
Field[] destFields = destClass.getDeclaredFields();
for (Field destField : destFields) {
String name = destField.getName();
try {
Field sourceField = sourceClass.getDeclaredField(name);
destField.setAccessible(true);
sourceField.setAccessible(true);
Object sourceObj = sourceField.get(source);
if (sourceObj instanceof Map) {
Object destObj = destField.get(dest);
if (destObj == null) {
destField.set(dest, sourceObj);
} else if (destObj instanceof Map) {
@SuppressWarnings("unchecked")
Map<Object, Object> sourceMap = (Map<Object, Object>) sourceObj;
@SuppressWarnings("unchecked")
Map<Object, Object> destMap = (Map<Object, Object>) destObj;
destMap.clear();
Iterator<?> entries = sourceMap.entrySet().iterator();
while (entries.hasNext()) {
Map.Entry<?, ?> entry = (Map.Entry<?, ?>)entries.next();
Object key = entry.getKey();
Object value = entry.getValue();
destMap.put(key, value);
}
} else {
logger.error("Incompatible field type! '{}'", name);
}
} else {
destField.set(dest, sourceObj);
}
} catch (Exception e) {
logger.error("Unable to copy field: {}", name, e);
}
}
destClass = destClass.getSuperclass();
sourceClass = sourceClass.getSuperclass();
}
}
private volatile long milliSeconds = 3000L;
private volatile DelayTimer timer;
private final ByteArrayWrapper defaultKey;
private final StorageBase storage;
private AtomicInteger references = new AtomicInteger(1);
private final ReentrantLock actionLock = new ReentrantLock();
private volatile Map<ByteArrayWrapper, Object> actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
private AtomicBoolean isOpen = new AtomicBoolean(false);
/**
* Creates or opens a new database file.
*/
private Storage(File storageFile) throws IOException {
this.storage = new StorageBase(storageFile);
this.defaultKey = wrap("");
this.timer = new DelayTimer("Storage Writer", false, new DelayTimer.Callback() {
@Override
public void execute() {
Map<ByteArrayWrapper, Object> actions = Storage.this.actionMap;
ReentrantLock actionLock2 = Storage.this.actionLock;
try {
actionLock2.lock();
// do a fast swap on the actionMap.
Storage.this.actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
} finally {
actionLock2.unlock();
}
Storage.this.storage.doActionThings(actions);
}
});
this.isOpen.set(true);
}
/**
* Returns the number of objects in the database.
* <p>
* SLOW because this must save all data to disk first!
*/
public final int size() {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
// flush actions
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
return this.storage.size();
}
/**
* Checks if there is a object corresponding to the given key.
*/
public final boolean contains(String key) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
return this.storage.contains(wrap(key));
}
/**
* Reads a object using the default (blank) key, and casts it to the expected class
*/
public final <T> T get() {
return get0(this.defaultKey);
}
/**
* Reads a object using the specific key, and casts it to the expected class
*/
public final <T> T get(String key) {
ByteArrayWrapper wrap = wrap(key);
return get0(wrap);
}
/**
* Copies the saved data (from) into the passed-in data. This just assigns all of the values from one to the other.
* <p>
* This will check to see if there is an associated key for that data, if not - it will use the default (blank)
*
* @param data The data that will hold the copy of the data from disk
*/
public void load(Object data) {
Object source = get();
if (source != null) {
Storage.copyFields(source, data);
}
}
/**
* Copies the saved data (from) into the passed-in data. This just assigns all of the values from one to the other.
* <p>
* The key/data will be associated for the lifetime of the object.
*
* @param object The object that will hold the copy of the data from disk (but not change the reference) once this is completed
*/
public void load(String key, Object object) {
ByteArrayWrapper wrap = wrap(key);
Object source = get0(wrap);
if (source != null) {
Storage.copyFields(source, object);
}
}
/**
* Reads a object from pending or from storage
*/
private final <T> T get0(ByteArrayWrapper key) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
// if the object in is pending, we get it from there
try {
this.actionLock.lock();
Object object = this.actionMap.get(key);
if (object != null) {
@SuppressWarnings("unchecked")
T returnObject = (T) object;
return returnObject;
}
} finally {
this.actionLock.unlock();
}
// not found, so we have to go find it on disk
return this.storage.get(key);
}
/**
* Save the storage to disk, once xxxx milli-seconds have passed.
* This is to help prevent thrashing the disk, or wearing it out on multiple, rapid, changes.
* <p>
* This will save the ALL of the pending save actions to the file
*/
public final void saveNow() {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
}
/**
* Saves the given data to storage with the associated key.
* <p>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
public final void save(String key, Object object) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
ByteArrayWrapper wrap = wrap(key);
action(wrap, object);
// timer action runs on TIMER thread, not this thread
this.timer.delay(this.milliSeconds);
}
/**
* Saves the given object to storage with the associated key.
* <p>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
public final void saveNow(String key, Object object) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
ByteArrayWrapper wrap = wrap(key);
action(wrap, object);
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
}
/**
* Adds the given object to the storage using a default (blank) key, OR -- if it has been registered, using it's registered key
* <p>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
public final void save(Object object) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
action(this.defaultKey, object);
// timer action runs on TIMER thread, not this thread
this.timer.delay(this.milliSeconds);
}
/**
* Adds the given object to the storage using a default (blank) key
* <p>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
public final void saveNow(Object object) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
action(this.defaultKey, object);
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
}
/**
* Deletes an object from storage. To ALSO remove from the cache, use unRegister(key)
*
* @return true if the delete was successful. False if there were problems deleting the data.
*/
public final boolean delete(String key) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
ByteArrayWrapper wrap = wrap(key);
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
return this.storage.delete(wrap);
}
/**
* Closes the database and file.
*/
private final void close() {
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
// have to "close" it after we run the timer!
this.isOpen.set(false);
this.storage.close();
}
/**
* @return the file that backs this storage
*/
public final File getFile() {
return this.storage.getFile();
}
/**
* Gets the backing file size.
*
* @return -1 if there was an error
*/
public final long getFileSize() {
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
return this.storage.getFileSize();
}
/**
* @return true if there are objects queued to be written?
*/
public final boolean hasWriteWaiting() {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
return this.timer.isWaiting();
}
/**
* @param delay milliseconds to wait
*/
public final void setSaveDelay(long milliSeconds) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
this.milliSeconds = milliSeconds;
}
/**
* @return the delay in milliseconds this will wait after the last action to flush the data to the disk
*/
public final long getSaveDelay() {
return this.milliSeconds;
}
/**
* @return the version of data stored in the database
*/
public final int getVersion() {
return this.storage.getVersion();
}
/**
* Sets the version of data stored in the database
*/
public final void setVersion(int version) {
this.storage.setVersion(version);
}
private final ByteArrayWrapper wrap(String key) {
byte[] bytes = key.getBytes(OS.UTF_8);
SHA256Digest digest = new SHA256Digest();
digest.update(bytes, 0, bytes.length);
byte[] hashBytes = new byte[digest.getDigestSize()];
digest.doFinal(hashBytes, 0);
ByteArrayWrapper wrap = ByteArrayWrapper.wrap(hashBytes);
return wrap;
}
private void action(ByteArrayWrapper key, Object object) {
try {
this.actionLock.lock();
// push action to map
this.actionMap.put(key, object);
} finally {
this.actionLock.unlock();
}
}
private final void increaseReference() {
this.references.incrementAndGet();
}
/** return true when this is the last reference */
private final boolean decrementReference() {
return this.references.decrementAndGet() == 0;
}
}

213
Dorkbox-Util/src/dorkbox/util/storage/StorageBase.java
View File

@ -15,16 +15,16 @@
*/
package dorkbox.util.storage;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import com.esotericsoftware.kryo.KryoException;
import com.esotericsoftware.kryo.io.Output;
import dorkbox.util.SerializationManager;
import dorkbox.util.bytes.ByteArrayWrapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.lang.ref.WeakReference;
import java.nio.channels.FileLock;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
@ -35,45 +35,39 @@ import java.util.zip.DeflaterOutputStream;
import java.util.zip.Inflater;
import java.util.zip.InflaterInputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.KryoException;
import com.esotericsoftware.kryo.io.Output;
import dorkbox.util.bytes.ByteArrayWrapper;
// a note on file locking between c and java
// http://panks-dev.blogspot.de/2008/04/linux-file-locks-java-and-others.html
// Also, file locks on linux are ADVISORY. if an app doesn't care about locks, then it can do stuff -- even if locked by another app
public class StorageBase {
@SuppressWarnings("unused")
public
class StorageBase {
private final Logger logger = LoggerFactory.getLogger(getClass());
// File pointer to the data start pointer header.
private static final long VERSION_HEADER_LOCATION = 0;
private static final long VERSION_HEADER_LOCATION = 0;
// File pointer to the num records header.
private static final long NUM_RECORDS_HEADER_LOCATION = 4;
// File pointer to the data start pointer header.
private static final long DATA_START_HEADER_LOCATION = 8;
private static final long DATA_START_HEADER_LOCATION = 8;
// Total length in bytes of the global database headers.
static final int FILE_HEADERS_REGION_LENGTH = 16;
static final int FILE_HEADERS_REGION_LENGTH = 16;
// The in-memory index (for efficiency, all of the record info is cached in memory).
private final Map<ByteArrayWrapper, Metadata> memoryIndex;
// determines how much the index will grow by
private Float weight;
private final Float weight;
// The keys are weak! When they go, the map entry is removed!
private final ReentrantLock referenceLock = new ReentrantLock();
private volatile Map<ByteArrayWrapper, Object> referencePendingWrite = new HashMap<ByteArrayWrapper, Object>();
private final File baseFile;
@ -97,32 +91,29 @@ public class StorageBase {
// save references to these, so they don't have to be created/destroyed any time there is I/O
private final Kryo kryo;
private final SerializationManager serializationManager;
private Deflater deflater;
private DeflaterOutputStream outputStream;
private final Deflater deflater;
private final DeflaterOutputStream outputStream;
private Inflater inflater;
private InflaterInputStream inputStream;
private final Inflater inflater;
private final InflaterInputStream inputStream;
/**
* Creates or opens a new database file.
*/
StorageBase(String filePath) throws IOException {
this(new File(filePath));
}
StorageBase(File filePath, SerializationManager serializationManager) throws IOException {
this.serializationManager = serializationManager;
/**
* Creates or opens a new database file.
*/
StorageBase(File filePath) throws IOException {
this.logger.info("Opening storage file: '{}'", filePath.getAbsolutePath());
this.baseFile = filePath;
File parentFile = this.baseFile.getParentFile();
if (parentFile != null) {
parentFile.mkdirs();
if (parentFile != null && !parentFile.exists()) {
if (!parentFile.mkdirs()) {
throw new IOException("Unable to create dirs for: " + filePath);
}
}
this.file = new RandomAccessFile(this.baseFile, "rw");
@ -132,7 +123,8 @@ public class StorageBase {
this.databaseVersion = this.file.readInt();
this.numberOfRecords = this.file.readInt();
this.dataPosition = this.file.readLong();
} else {
}
else {
setVersionNumber(this.file, 0);
// always start off with 4 records
@ -149,18 +141,16 @@ public class StorageBase {
throw new IllegalArgumentException("Unable to parse header information from storage. Maybe it's corrupted?");
}
//noinspection AutoBoxing
this.logger.info("Storage version: {}", this.databaseVersion);
this.kryo = new Kryo();
this.kryo.setRegistrationRequired(false);
this.deflater = new Deflater(7, true);
this.outputStream = new DeflaterOutputStream(new FileOutputStream(this.file.getFD()), this.deflater, 65536, true);
this.inflater = new Inflater(true);
this.inputStream = new InflaterInputStream(new FileInputStream(this.file.getFD()), this.inflater, 65536);
//noinspection AutoBoxing
this.weight = .5F;
this.memoryIndex = new ConcurrentHashMap<ByteArrayWrapper, Metadata>(this.numberOfRecords);
@ -183,7 +173,8 @@ public class StorageBase {
/**
* Returns the current number of records in the database.
*/
final int size() {
final
int size() {
// wrapper flushes first (protected by lock)
// not protected by lock
return this.memoryIndex.size();
@ -192,21 +183,19 @@ public class StorageBase {
/**
* Checks if there is a record belonging to the given key.
*/
final boolean contains(ByteArrayWrapper key) {
final
boolean contains(ByteArrayWrapper key) {
// protected by lock
// check to see if it's in the pending ops
if (this.referencePendingWrite.containsKey(key)) {
return true;
}
return this.memoryIndex.containsKey(key);
}
/**
* @return an object for a specified key ONLY FROM THE REFERENCE CACHE
*/
final <T> T getCached(ByteArrayWrapper key) {
final
<T> T getCached(ByteArrayWrapper key) {
// protected by lock
Metadata meta = this.memoryIndex.get(key);
@ -223,7 +212,7 @@ public class StorageBase {
if (ref != null) {
@SuppressWarnings("unchecked")
T referenceObject = (T) ref.get();
T referenceObject = (T) ref.get();
return referenceObject;
}
} finally {
@ -236,7 +225,8 @@ public class StorageBase {
/**
* @return an object for a specified key form referenceCache FIRST, then from DISK
*/
final <T> T get(ByteArrayWrapper key) {
final
<T> T get(ByteArrayWrapper key) {
// NOT protected by lock
Metadata meta = this.memoryIndex.get(key);
@ -253,7 +243,7 @@ public class StorageBase {
if (ref != null) {
@SuppressWarnings("unchecked")
T referenceObject = (T) ref.get();
T referenceObject = (T) ref.get();
return referenceObject;
}
} finally {
@ -266,7 +256,7 @@ public class StorageBase {
this.inflater.reset();
this.file.seek(meta.dataPointer);
T readRecordData = meta.readData(this.kryo, this.inputStream);
T readRecordData = Metadata.readData(this.serializationManager, this.inputStream);
if (readRecordData != null) {
// now stuff it into our reference cache for future lookups!
@ -281,10 +271,10 @@ public class StorageBase {
return readRecordData;
} catch (KryoException e) {
this.logger.error("Error while geting data from disk. Ignoring previous value.");
this.logger.error("Error while getting data from disk. Ignoring previous value.");
return null;
} catch (Exception e) {
this.logger.error("Error while geting data from disk", e);
this.logger.error("Error while getting data from disk", e);
return null;
}
}
@ -294,7 +284,8 @@ public class StorageBase {
*
* @return true if the delete was successful. False if there were problems deleting the data.
*/
final boolean delete(ByteArrayWrapper key) {
final
boolean delete(ByteArrayWrapper key) {
// pending ops flushed (protected by lock)
// not protected by lock
Metadata delRec = this.memoryIndex.get(key);
@ -312,11 +303,13 @@ public class StorageBase {
/**
* Closes the database and file.
*/
final void close() {
final
void close() {
// pending ops flushed (protected by lock)
// not protected by lock
try {
this.file.getFD().sync();
this.file.getFD()
.sync();
this.file.close();
this.memoryIndex.clear();
@ -344,22 +337,22 @@ public class StorageBase {
/**
* @return the file that backs this storage
*/
final File getFile() {
final
File getFile() {
return this.baseFile;
}
/**
* Saves the given data to storage.
* <p>
* <p/>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
* <p>
* <p/>
* Will also save the object in a cache.
*
* @return the metadata for the saved object
*/
private final void save0(ByteArrayWrapper key, Object object, DeflaterOutputStream fileOutputStream, Deflater deflater) {
private
void save0(ByteArrayWrapper key, Object object, DeflaterOutputStream fileOutputStream, Deflater deflater) {
deflater.reset();
Metadata metaData = this.memoryIndex.get(key);
@ -370,19 +363,23 @@ public class StorageBase {
try {
if (currentRecordCount == 1) {
// if we are the ONLY one, then we can do things differently.
// just dump the data agian to disk.
FileLock lock = this.file.getChannel().lock(this.dataPosition, Long.MAX_VALUE-this.dataPosition, false); // don't know how big it is, so max value it
// just dump the data again to disk.
FileLock lock = this.file.getChannel()
.lock(this.dataPosition,
Long.MAX_VALUE - this.dataPosition,
false); // don't know how big it is, so max value it
this.file.seek(this.dataPosition); // this is the end of the file, we know this ahead-of-time
metaData.writeDataFast(this.kryo, object, fileOutputStream);
Metadata.writeDataFast(this.serializationManager, object, file, fileOutputStream);
// have to re-specify the capacity and size
int sizeOfWrittenData = (int) (this.file.length() - this.dataPosition);
metaData.dataCapacity = sizeOfWrittenData;
metaData.dataCount = sizeOfWrittenData;
lock.release();
} else {
}
else {
// this is comparatively slow, since we serialize it first to get the size, then we put it in the file.
ByteArrayOutputStream dataStream = getDataAsByteArray(this.kryo, object, deflater);
ByteArrayOutputStream dataStream = getDataAsByteArray(this.serializationManager, object, deflater);
int size = dataStream.size();
if (size > metaData.dataCapacity) {
@ -404,10 +401,11 @@ public class StorageBase {
} catch (IOException e) {
this.logger.error("Error while saving data to disk", e);
}
} else {
}
else {
try {
// try to move the read head in order
setRecordCount(this.file, currentRecordCount+1);
setRecordCount(this.file, currentRecordCount + 1);
// This will make sure that there is room to write a new record. This is zero indexed.
// this will skip around if moves occur
@ -425,9 +423,10 @@ public class StorageBase {
// save out the data. Because we KNOW that we are writing this to the end of the file,
// there are some tricks we can use.
FileLock lock = this.file.getChannel().lock(length, Long.MAX_VALUE-length, false); // don't know how big it is, so max value it
FileLock lock = this.file.getChannel()
.lock(length, Long.MAX_VALUE - length, false); // don't know how big it is, so max value it
this.file.seek(length); // this is the end of the file, we know this ahead-of-time
metaData.writeDataFast(this.kryo, object, fileOutputStream);
Metadata.writeDataFast(this.serializationManager, object, file, fileOutputStream);
lock.release();
metaData.dataCount = deflater.getTotalOut();
@ -445,7 +444,8 @@ public class StorageBase {
}
private ByteArrayOutputStream getDataAsByteArray(Kryo kryo, Object data, Deflater deflater) throws IOException {
private static
ByteArrayOutputStream getDataAsByteArray(SerializationManager kryo, Object data, Deflater deflater) throws IOException {
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
OutputStream outputStream = new DeflaterOutputStream(byteArrayOutputStream, deflater);
Output output = new Output(outputStream, 1024); // write 1024 at a time
@ -484,23 +484,27 @@ public class StorageBase {
/**
* "intelligent" move strategy.
*
* <p/>
* we should increase by some weight (ie: .5) would increase the number of allocated
* record headers by 50%, instead of just incrementing them by one at a time.
*/
private final int getWeightedNewRecordCount(int numberOfRecords) {
int newNumberOfRecords = numberOfRecords + 1 + (int) (numberOfRecords * this.weight); // int used for rounding
return newNumberOfRecords;
private
int getWeightedNewRecordCount(int numberOfRecords) {
//noinspection AutoUnboxing
return numberOfRecords + 1 + (int) (numberOfRecords * this.weight);
}
private void deleteRecordData(Metadata deletedRecord, int sizeOfDataToAdd) throws IOException {
private
void deleteRecordData(Metadata deletedRecord, int sizeOfDataToAdd) throws IOException {
if (this.file.length() == deletedRecord.dataPointer + deletedRecord.dataCapacity) {
// shrink file since this is the last record in the file
FileLock lock = this.file.getChannel().lock(deletedRecord.dataPointer, Long.MAX_VALUE-deletedRecord.dataPointer, false);
FileLock lock = this.file.getChannel()
.lock(deletedRecord.dataPointer, Long.MAX_VALUE - deletedRecord.dataPointer, false);
this.file.setLength(deletedRecord.dataPointer);
lock.release();
} else {
}
else {
// we MIGHT be the FIRST record
Metadata first = index_getMetaDataFromData(this.dataPosition);
if (first == deletedRecord) {
@ -515,22 +519,25 @@ public class StorageBase {
long endIndexPointer = Metadata.getMetaDataPointer(newNumberOfRecords);
long endOfDataPointer = deletedRecord.dataPointer + deletedRecord.dataCapacity;
long newEndOfDataPointer = endOfDataPointer-sizeOfDataToAdd;
long newEndOfDataPointer = endOfDataPointer - sizeOfDataToAdd;
if (endIndexPointer < this.dataPosition && endIndexPointer <= newEndOfDataPointer) {
// one option is to shrink the RECORD section to fit the new data
setDataPosition(this.file, newEndOfDataPointer);
} else {
}
else {
// option two is to grow the RECORD section, and put the data at the end of the file
setDataPosition(this.file, endOfDataPointer);
}
} else {
}
else {
Metadata previous = index_getMetaDataFromData(deletedRecord.dataPointer - 1);
if (previous != null) {
// append space of deleted record onto previous record
previous.dataCapacity += deletedRecord.dataCapacity;
previous.writeDataInfo(this.file);
} else {
}
else {
// because there is no "previous", that means we MIGHT be the FIRST record
// well, we're not the first record. which one is RIGHT before us?
// it should be "previous", so something fucked up
@ -540,11 +547,14 @@ public class StorageBase {
}
}
private void deleteRecordIndex(ByteArrayWrapper key, Metadata deleteRecord) throws IOException {
private
void deleteRecordIndex(ByteArrayWrapper key, Metadata deleteRecord) throws IOException {
int currentNumRecords = this.memoryIndex.size();
if (deleteRecord.indexPosition != currentNumRecords - 1) {
Metadata last = Metadata.readHeader(this.file, currentNumRecords - 1);
assert last != null;
last.move(this.file, deleteRecord.indexPosition);
}
@ -557,10 +567,12 @@ public class StorageBase {
/**
* Writes the number of records header to the file.
*/
private final void setVersionNumber(RandomAccessFile file, int versionNumber) throws IOException {
private
void setVersionNumber(RandomAccessFile file, int versionNumber) throws IOException {
this.databaseVersion = versionNumber;
FileLock lock = this.file.getChannel().lock(VERSION_HEADER_LOCATION, 4, false);
FileLock lock = this.file.getChannel()
.lock(VERSION_HEADER_LOCATION, 4, false);
file.seek(VERSION_HEADER_LOCATION);
file.writeInt(versionNumber);
lock.release();
@ -569,10 +581,12 @@ public class StorageBase {
/**
* Writes the number of records header to the file.
*/
private final void setRecordCount(RandomAccessFile file, int numberOfRecords) throws IOException {
private
void setRecordCount(RandomAccessFile file, int numberOfRecords) throws IOException {
this.numberOfRecords = numberOfRecords;
FileLock lock = this.file.getChannel().lock(NUM_RECORDS_HEADER_LOCATION, 4, false);
FileLock lock = this.file.getChannel()
.lock(NUM_RECORDS_HEADER_LOCATION, 4, false);
file.seek(NUM_RECORDS_HEADER_LOCATION);
file.writeInt(numberOfRecords);
lock.release();
@ -581,10 +595,12 @@ public class StorageBase {
/**
* Writes the data start position to the file.
*/
private final void setDataPosition(RandomAccessFile file, long dataPositionPointer) throws IOException {
private
void setDataPosition(RandomAccessFile file, long dataPositionPointer) throws IOException {
this.dataPosition = dataPositionPointer;
FileLock lock = this.file.getChannel().lock(DATA_START_HEADER_LOCATION, 8, false);
FileLock lock = this.file.getChannel()
.lock(DATA_START_HEADER_LOCATION, 8, false);
file.seek(DATA_START_HEADER_LOCATION);
file.writeLong(dataPositionPointer);
lock.release();
@ -608,9 +624,12 @@ public class StorageBase {
* of the record data of the RecordHeader which is returned. Returns null if the location is not part of a record.
* (O(n) mem accesses)
*/
private final Metadata index_getMetaDataFromData(long targetFp) {
Iterator<Metadata> iterator = this.memoryIndex.values().iterator();
private
Metadata index_getMetaDataFromData(long targetFp) {
Iterator<Metadata> iterator = this.memoryIndex.values()
.iterator();
//noinspection WhileLoopReplaceableByForEach
while (iterator.hasNext()) {
Metadata next = iterator.next();
if (targetFp >= next.dataPointer && targetFp < next.dataPointer + next.dataCapacity) {
@ -623,11 +642,13 @@ public class StorageBase {
/**
* Ensure index capacity. This operation makes sure the INDEX REGION is large enough to accommodate additional entries.
* Ensure index capacity. This operation makes sure the INDEX REGION is large enough to accommodate additional entries.
*/
private final void ensureIndexCapacity(RandomAccessFile file) throws IOException {
private
void ensureIndexCapacity(RandomAccessFile file) throws IOException {
int numberOfRecords = this.numberOfRecords; // because we are zero indexed, this is ALSO the index where the record will START
long endIndexPointer = Metadata.getMetaDataPointer(numberOfRecords + 1); // +1 because this is where that index will END (the start of the NEXT one)
long endIndexPointer = Metadata.getMetaDataPointer(
numberOfRecords + 1); // +1 because this is where that index will END (the start of the NEXT one)
// just set the data position to the end of the file, since we don't have any data yet.
if (endIndexPointer > file.length() && numberOfRecords == 0) {

450
Dorkbox-Util/test/dorkbox/util/StorageTest.java
View File

@ -1,44 +1,118 @@
package dorkbox.util;
import static org.junit.Assert.fail;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Registration;
import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import dorkbox.util.storage.DiskStorageIfface;
import dorkbox.util.storage.MakeStorage;
import io.netty.buffer.ByteBuf;
import org.junit.*;
import org.junit.runners.MethodSorters;
import java.io.File;
import java.io.IOException;
import java.io.OptionalDataException;
import java.util.Arrays;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runners.MethodSorters;
import dorkbox.util.storage.Storage;
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
public class StorageTest {
public
class StorageTest {
private static final String TEST_DB = "sampleFile.records";
private static final File TEST_DB = new File("sampleFile.records");
private static final SerializationManager manager = new SerializationManager() {
Kryo kryo = new Kryo();
static void log(String s) {
@Override
public
boolean setReferences(final boolean references) {
kryo.setReferences(references);
return false;
}
@Override
public
void setRegistrationRequired(final boolean registrationRequired) {
kryo.setRegistrationRequired(registrationRequired);
}
@Override
public
void register(final Class<?> clazz) {
kryo.register(clazz);
}
@Override
public
void register(final Class<?> clazz, final Serializer<?> serializer) {
kryo.register(clazz, serializer);
}
@Override
public
Registration register(final Class<?> type, final Serializer<?> serializer, final int id) {
kryo.register(type, serializer, id);
return null;
}
@Override
public
void write(final ByteBuf buffer, final Object message) {
final Output output = new Output();
writeClassAndObject(output, message);
buffer.writeBytes(output.getBuffer());
}
@Override
public
Object read(final ByteBuf buffer, final int length) {
final Input input = new Input();
buffer.readBytes(input.getBuffer());
final Object o = readClassAndObject(input);
buffer.skipBytes(input.position());
return o;
}
@Override
public
void writeClassAndObject(final Output output, final Object value) {
kryo.writeClassAndObject(output, value);
}
@Override
public
Object readClassAndObject(final Input input) {
return kryo.readClassAndObject(input);
}
};
static
void log(String s) {
System.err.println(s);
}
@Before
public void deleteDB() {
Storage.delete(new File(TEST_DB));
public
void deleteDB() {
MakeStorage.delete(TEST_DB);
}
@After
public void delete2DB() {
Storage.delete(new File(TEST_DB));
public
void delete2DB() {
MakeStorage.delete(TEST_DB);
}
@Test
public void testCreateDB() throws IOException {
Storage storage = Storage.open(TEST_DB);
public
void testCreateDB() throws IOException {
DiskStorageIfface storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
int numberOfRecords1 = storage.size();
long size1 = storage.getFileSize();
@ -46,67 +120,84 @@ public class StorageTest {
Assert.assertEquals("count is not correct", numberOfRecords1, 0);
Assert.assertEquals("size is not correct", size1, 208L); // NOTE this will change based on the data size added!
Storage.close(storage);
MakeStorage.close(storage);
storage = Storage.open(TEST_DB);
storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
int numberOfRecords2 = storage.size();
long size2 = storage.getFileSize();
Assert.assertEquals("Record count is not the same", numberOfRecords1, numberOfRecords2);
Assert.assertEquals("size is not the same", size1, size2);
Storage.close(storage);
MakeStorage.close(storage);
}
@Test
public void testAddAsOne() throws IOException, ClassNotFoundException {
int total = 100;
try {
Storage storage = Storage.open(TEST_DB);
for (int i=0;i<total;i++) {
add(storage, i);
}
Storage.close(storage);
storage = Storage.open(TEST_DB);
for (int i=0;i<total;i++) {
String record1Data = createData(i);
String readRecord = readRecord(storage, i);
Assert.assertEquals("Object is not the same", record1Data, readRecord);
}
Storage.close(storage);
} catch (Exception e) {
e.printStackTrace();
fail("Error!");
}
}
@Test
public void testAddNoKeyRecords() throws IOException, ClassNotFoundException {
public
void testAddAsOne() throws IOException, ClassNotFoundException {
int total = 100;
try {
Storage storage = Storage.open(TEST_DB);
for (int i=0;i<total;i++) {
DiskStorageIfface storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
for (int i = 0; i < total; i++) {
add(storage, i);
}
MakeStorage.close(storage);
storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
for (int i = 0; i < total; i++) {
String record1Data = createData(i);
String readRecord = readRecord(storage, i);
Assert.assertEquals("Object is not the same", record1Data, readRecord);
}
MakeStorage.close(storage);
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Error!");
}
}
@Test
public
void testAddNoKeyRecords() throws IOException, ClassNotFoundException {
int total = 100;
try {
DiskStorageIfface storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
for (int i = 0; i < total; i++) {
log("adding record " + i + "...");
String addRecord = createData(i);
storage.save(addRecord);
storage.put(addRecord);
log("reading record " + i + "...");
String readData = storage.get();
Assert.assertEquals("Object is not the same", addRecord, readData);
}
Storage.close(storage);
MakeStorage.close(storage);
storage = Storage.open(TEST_DB);
storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
String dataCheck = createData(total-1);
log("reading record " + (total-1) + "...");
String dataCheck = createData(total - 1);
log("reading record " + (total - 1) + "...");
String readData = storage.get();
Assert.assertEquals("Object is not the same", dataCheck, readData);
@ -117,101 +208,123 @@ public class StorageTest {
Assert.assertEquals("count is not correct", numberOfRecords1, 1);
Assert.assertEquals("size is not correct", size1, 235L); // NOTE this will change based on the data size added!
Storage.close(storage);
MakeStorage.close(storage);
} catch (Exception e) {
e.printStackTrace();
fail("Error!");
Assert.fail("Error!");
}
}
@Test
public void testAddRecords_DelaySaveA() throws IOException, ClassNotFoundException {
public
void testAddRecords_DelaySaveA() throws IOException, ClassNotFoundException {
int total = 100;
try {
Storage storage = Storage.open(TEST_DB);
for (int i=0;i<total;i++) {
DiskStorageIfface storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
for (int i = 0; i < total; i++) {
add(storage, i);
}
synchronized (Thread.currentThread()) {
Thread.currentThread().wait(storage.getSaveDelay() + 1000L);
Thread.currentThread()
.wait(storage.getSaveDelay() + 1000L);
}
for (int i=0;i<total;i++) {
for (int i = 0; i < total; i++) {
String record1Data = createData(i);
String readRecord = readRecord(storage, i);
Assert.assertEquals("Object is not the same", record1Data, readRecord);
}
Storage.close(storage);
MakeStorage.close(storage);
storage = Storage.open(TEST_DB);
for (int i=0;i<total;i++) {
storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
for (int i = 0; i < total; i++) {
String dataCheck = createData(i);
String readRecord = readRecord(storage, i);
Assert.assertEquals("Object is not the same", dataCheck, readRecord);
}
Storage.close(storage);
MakeStorage.close(storage);
} catch (Exception e) {
e.printStackTrace();
fail("Error!");
Assert.fail("Error!");
}
}
@Test
public void testAddRecords_DelaySaveB() throws IOException, ClassNotFoundException {
public
void testAddRecords_DelaySaveB() throws IOException, ClassNotFoundException {
int total = 100;
try {
Storage storage = Storage.open(TEST_DB);
for (int i=0;i<total;i++) {
DiskStorageIfface storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
for (int i = 0; i < total; i++) {
add(storage, i);
}
for (int i=0;i<total;i++) {
for (int i = 0; i < total; i++) {
String record1Data = createData(i);
String readRecord = readRecord(storage, i);
Assert.assertEquals("Object is not the same", record1Data, readRecord);
}
Storage.close(storage);
MakeStorage.close(storage);
storage = Storage.open(TEST_DB);
for (int i=0;i<total;i++) {
storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
for (int i = 0; i < total; i++) {
String dataCheck = createData(i);
String readRecord = readRecord(storage, i);
Assert.assertEquals("Object is not the same", dataCheck, readRecord);
}
Storage.close(storage);
MakeStorage.close(storage);
} catch (Exception e) {
e.printStackTrace();
fail("Error!");
Assert.fail("Error!");
}
}
@Test
public void testLoadRecords() throws IOException, ClassNotFoundException {
public
void testLoadRecords() throws IOException, ClassNotFoundException {
int total = 100;
try {
Storage storage = Storage.open(TEST_DB);
for (int i=0;i<total;i++) {
DiskStorageIfface storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
for (int i = 0; i < total; i++) {
String addRecord = add(storage, i);
String readRecord = readRecord(storage, i);
Assert.assertEquals("Object is not the same", addRecord, readRecord);
}
Storage.close(storage);
MakeStorage.close(storage);
storage = Storage.open(TEST_DB);
for (int i=0;i<total;i++) {
storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
for (int i = 0; i < total; i++) {
String dataCheck = createData(i);
String readRecord = readRecord(storage, i);
@ -223,43 +336,53 @@ public class StorageTest {
String createKey = createKey(63);
makeData(data);
storage.save(createKey, data);
storage.put(createKey, data);
Data data2 = new Data();
storage.load(createKey, data2);
Assert.assertEquals("Object is not the same", data, data2);
Storage.close(storage);
storage = Storage.open(TEST_DB);
MakeStorage.close(storage);
storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
data2 = new Data();
storage.load(createKey, data2);
Assert.assertEquals("Object is not the same", data, data2);
Storage.close(storage);
MakeStorage.close(storage);
} catch (Exception e) {
e.printStackTrace();
fail("Error!");
Assert.fail("Error!");
}
}
@Test
public void testAddRecordsDelete1Record() throws IOException, ClassNotFoundException {
public
void testAddRecordsDelete1Record() throws IOException, ClassNotFoundException {
int total = 100;
try {
Storage storage = Storage.open(TEST_DB);
for (int i=0;i<total;i++) {
DiskStorageIfface storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
for (int i = 0; i < total; i++) {
String addRecord = add(storage, i);
String readRecord = readRecord(storage, i);
Assert.assertEquals("Object is not the same", addRecord, readRecord);
}
Storage.close(storage);
MakeStorage.close(storage);
storage = Storage.open(TEST_DB);
for (int i=0;i<total;i++) {
storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
for (int i = 0; i < total; i++) {
String dataCheck = createData(i);
String readRecord = readRecord(storage, i);
@ -276,7 +399,7 @@ public class StorageTest {
if (storage.contains(createKey(3))) {
fail("record NOT successfully deleted.");
Assert.fail("record NOT successfully deleted.");
}
// now we add 3 back
@ -285,9 +408,12 @@ public class StorageTest {
Assert.assertEquals("Object is not the same", dataCheck, addRecord);
Storage.close(storage);
MakeStorage.close(storage);
storage = Storage.open(TEST_DB);
storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
// check 9 again
readRecord = readRecord(storage, 9);
@ -301,110 +427,134 @@ public class StorageTest {
} catch (Exception e) {
e.printStackTrace();
fail("Error!");
Assert.fail("Error!");
}
}
@Test
public void testUpdateRecords() throws IOException, ClassNotFoundException {
public
void testUpdateRecords() throws IOException, ClassNotFoundException {
int total = 100;
try {
Storage storage = Storage.open(TEST_DB);
for (int i=0;i<total;i++) {
DiskStorageIfface storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
for (int i = 0; i < total; i++) {
String addRecord = add(storage, i);
String readRecord = readRecord(storage, i);
Assert.assertEquals("Object is not the same", addRecord, readRecord);
}
Storage.close(storage);
MakeStorage.close(storage);
storage = Storage.open(TEST_DB);
storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
String updateRecord = updateRecord(storage, 3, createData(3) + "new");
String readRecord = readRecord(storage, 3);
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
Storage.close(storage);
storage = Storage.open(TEST_DB);
MakeStorage.close(storage);
storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
readRecord = readRecord(storage, 3);
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
updateRecord = updateRecord(storage, 3, createData(3));
Storage.close(storage);
storage = Storage.open(TEST_DB);
MakeStorage.close(storage);
storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
readRecord = readRecord(storage, 3);
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
Storage.close(storage);
storage = Storage.open(TEST_DB);
MakeStorage.close(storage);
storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
updateRecord = updateRecord(storage, 0, createData(0) + "new");
readRecord = readRecord(storage, 0);
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
Storage.close(storage);
MakeStorage.close(storage);
} catch (Exception e) {
e.printStackTrace();
fail("Error!");
Assert.fail("Error!");
}
}
@Test
public void testSaveAllRecords() throws IOException, ClassNotFoundException {
public
void testSaveAllRecords() throws IOException, ClassNotFoundException {
int total = 100;
try {
Storage storage = Storage.open(TEST_DB);
for (int i=0;i<total;i++) {
DiskStorageIfface storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
for (int i = 0; i < total; i++) {
Data data = new Data();
makeData(data);
String createKey = createKey(i);
storage.save(createKey, data);
storage.put(createKey, data);
}
Storage.close(storage);
MakeStorage.close(storage);
Data data = new Data();
makeData(data);
storage = Storage.open(TEST_DB);
for (int i=0;i<total;i++) {
storage = MakeStorage.Disk()
.file(TEST_DB)
.serializer(manager)
.make();
for (int i = 0; i < total; i++) {
String createKey = createKey(i);
Data data2 = new Data();
storage.load(createKey, data2);
Assert.assertEquals("Object is not the same", data, data2);
}
Storage.close(storage);
MakeStorage.close(storage);
} catch (Exception e) {
e.printStackTrace();
fail("Error!");
Assert.fail("Error!");
}
}
private String createData(int number) {
private static
String createData(int number) {
return number + " data for record # " + number;
}
public String add(Storage storage, int number) throws IOException {
public static
String add(DiskStorageIfface storage, int number) throws IOException {
String record1Data = createData(number);
String record1Key = createKey(number);
log("adding record " + number + "...");
storage.save(record1Key, record1Data);
storage.put(record1Key, record1Data);
return record1Data;
}
public String readRecord(Storage storage, int number) throws OptionalDataException, ClassNotFoundException, IOException {
public static
String readRecord(DiskStorageIfface storage, int number) throws ClassNotFoundException, IOException {
String record1Key = createKey(number);
log("reading record " + number + "...");
@ -414,30 +564,34 @@ public class StorageTest {
return readData;
}
public void deleteRecord(Storage storage, int nNumber) throws OptionalDataException, ClassNotFoundException, IOException {
public static
void deleteRecord(DiskStorageIfface storage, int nNumber) throws ClassNotFoundException, IOException {
String record1Key = createKey(nNumber);
log("deleting record " + nNumber + "...");
storage.delete(record1Key);
}
private String updateRecord(Storage storage, int number, String newData) throws IOException {
private static
String updateRecord(DiskStorageIfface storage, int number, String newData) throws IOException {
String record1Key = createKey(number);
log("updating record " + number + "...");
storage.save(record1Key, newData);
storage.put(record1Key, newData);
return newData;
}
private String createKey(int number) {
private static
String createKey(int number) {
return "foo" + number;
}
// from kryo unit test.
private void makeData(Data data) {
StringBuilder buffer = new StringBuilder();
private static
void makeData(Data data) {
StringBuilder buffer = new StringBuilder(128);
for (int i = 0; i < 3; i++) {
buffer.append('a');
}
@ -446,8 +600,7 @@ public class StorageTest {
data.strings = new String[] {"ab012", "", null, "!@#$", "<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>"};
data.ints = new int[] {-1234567, 1234567, -1, 0, 1, Integer.MAX_VALUE, Integer.MIN_VALUE};
data.shorts = new short[] {-12345, 12345, -1, 0, 1, Short.MAX_VALUE, Short.MIN_VALUE};
data.floats = new float[] {0, -0, 1, -1, 123456, -123456, 0.1f, 0.2f, -0.3f, (float)Math.PI, Float.MAX_VALUE,
Float.MIN_VALUE};
data.floats = new float[] {0, -0, 1, -1, 123456, -123456, 0.1f, 0.2f, -0.3f, (float) Math.PI, Float.MAX_VALUE, Float.MIN_VALUE};
data.doubles = new double[] {0, -0, 1, -1, 123456, -123456, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE, Double.MIN_VALUE};
data.longs = new long[] {0, -0, 1, -1, 123456, -123456, 99999999999l, -99999999999l, Long.MAX_VALUE, Long.MIN_VALUE};
data.bytes = new byte[] {-123, 123, -1, 0, 1, Byte.MAX_VALUE, Byte.MIN_VALUE};
@ -455,17 +608,17 @@ public class StorageTest {
data.booleans = new boolean[] {true, false};
data.Ints = new Integer[] {-1234567, 1234567, -1, 0, 1, Integer.MAX_VALUE, Integer.MIN_VALUE};
data.Shorts = new Short[] {-12345, 12345, -1, 0, 1, Short.MAX_VALUE, Short.MIN_VALUE};
data.Floats = new Float[] {0f, -0f, 1f, -1f, 123456f, -123456f, 0.1f, 0.2f, -0.3f, (float)Math.PI, Float.MAX_VALUE,
Float.MIN_VALUE};
data.Doubles = new Double[] {0d, -0d, 1d, -1d, 123456d, -123456d, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE,
Double.MIN_VALUE};
data.Floats = new Float[] {0f, -0f, 1f, -1f, 123456f, -123456f, 0.1f, 0.2f, -0.3f, (float) Math.PI, Float.MAX_VALUE,
Float.MIN_VALUE};
data.Doubles = new Double[] {0d, -0d, 1d, -1d, 123456d, -123456d, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE, Double.MIN_VALUE};
data.Longs = new Long[] {0l, -0l, 1l, -1l, 123456l, -123456l, 99999999999l, -99999999999l, Long.MAX_VALUE, Long.MIN_VALUE};
data.Bytes = new Byte[] {-123, 123, -1, 0, 1, Byte.MAX_VALUE, Byte.MIN_VALUE};
data.Chars = new Character[] {32345, 12345, 0, 1, 63, Character.MAX_VALUE, Character.MIN_VALUE};
data.Booleans = new Boolean[] {true, false};
}
public static class Data {
public static
class Data {
public String string;
public String[] strings;
public int[] ints;
@ -485,11 +638,13 @@ public class StorageTest {
public Character[] Chars;
public Boolean[] Booleans;
public Data() {
public
Data() {
}
@Override
public int hashCode () {
public
int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + Arrays.hashCode(this.Booleans);
@ -514,7 +669,8 @@ public class StorageTest {
}
@Override
public boolean equals (Object obj) {
public
boolean equals(Object obj) {
if (this == obj) {
return true;
}
@ -524,7 +680,7 @@ public class StorageTest {
if (getClass() != obj.getClass()) {
return false;
}
Data other = (Data)obj;
Data other = (Data) obj;
if (!Arrays.equals(this.Booleans, other.Booleans)) {
return false;
}
@ -577,7 +733,8 @@ public class StorageTest {
if (other.string != null) {
return false;
}
} else if (!this.string.equals(other.string)) {
}
else if (!this.string.equals(other.string)) {
return false;
}
if (!Arrays.equals(this.strings, other.strings)) {
@ -587,7 +744,8 @@ public class StorageTest {
}
@Override
public String toString () {
public
String toString() {
return "Data";
}
}