wip - creating disk or memory based storage
This commit is contained in:
parent
927a5aece9
commit
08f7872a40
516
Dorkbox-Util/src/dorkbox/util/storage/DiskStorage.java
Normal file
516
Dorkbox-Util/src/dorkbox/util/storage/DiskStorage.java
Normal file
|
@ -0,0 +1,516 @@
|
|||
/*
|
||||
* Copyright 2014 dorkbox, llc
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package dorkbox.util.storage;
|
||||
|
||||
import dorkbox.util.DelayTimer;
|
||||
import dorkbox.util.SerializationManager;
|
||||
import dorkbox.util.bytes.ByteArrayWrapper;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
|
||||
/**
|
||||
* Nothing spectacular about this storage -- it allows for persistent storage of objects to disk.
|
||||
* <p/>
|
||||
* Be wary of opening the database file in different instances. Even with file-locks, you can corrupt the data.
|
||||
*/
|
||||
@SuppressWarnings("unused")
|
||||
public
|
||||
class DiskStorage implements DiskStorageIfface {
|
||||
private final DelayTimer timer;
|
||||
private final ByteArrayWrapper defaultKey;
|
||||
private final StorageBase storage;
|
||||
|
||||
private final AtomicInteger references = new AtomicInteger(1);
|
||||
private final ReentrantLock actionLock = new ReentrantLock();
|
||||
private final AtomicBoolean isOpen = new AtomicBoolean(false);
|
||||
private volatile long milliSeconds = 3000L;
|
||||
|
||||
private volatile Map<ByteArrayWrapper, Object> actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
|
||||
|
||||
/**
|
||||
* Creates or opens a new database file.
|
||||
*/
|
||||
protected
|
||||
DiskStorage(File storageFile, SerializationManager serializationManager) throws IOException {
|
||||
this.storage = new StorageBase(storageFile, serializationManager);
|
||||
this.defaultKey = ByteArrayWrapper.wrap("");
|
||||
|
||||
this.timer = new DelayTimer("Storage Writer", false, new DelayTimer.Callback() {
|
||||
@Override
|
||||
public
|
||||
void execute() {
|
||||
Map<ByteArrayWrapper, Object> actions = DiskStorage.this.actionMap;
|
||||
|
||||
ReentrantLock actionLock2 = DiskStorage.this.actionLock;
|
||||
|
||||
try {
|
||||
actionLock2.lock();
|
||||
|
||||
// do a fast swap on the actionMap.
|
||||
DiskStorage.this.actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
|
||||
} finally {
|
||||
actionLock2.unlock();
|
||||
}
|
||||
|
||||
DiskStorage.this.storage.doActionThings(actions);
|
||||
}
|
||||
});
|
||||
|
||||
this.isOpen.set(true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Returns the number of objects in the database.
|
||||
* <p/>
|
||||
* SLOW because this must save all data to disk first!
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
int size() {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
// flush actions
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
|
||||
return this.storage.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if there is a object corresponding to the given key.
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
boolean contains(String key) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
final ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
|
||||
// check if our pending actions has it, or if our storage index has it
|
||||
return this.actionMap.containsKey(wrap) || this.storage.contains(wrap);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a object using the default (blank) key, and casts it to the expected class
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
<T> T get() {
|
||||
return get0(this.defaultKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a object using the specific key, and casts it to the expected class
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
<T> T get(String key) {
|
||||
return get0(ByteArrayWrapper.wrap(key));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a object using the specific key, and casts it to the expected class
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
<T> T get(byte[] key) {
|
||||
return get0(ByteArrayWrapper.wrap(key));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a object using the specific key, and casts it to the expected class
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
<T> T get(ByteArrayWrapper key) {
|
||||
return get0(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Uses the DEFAULT key ("") to return saved data.
|
||||
* <p/>
|
||||
* This will check to see if there is an associated key for that data, if not - it will use data as the default
|
||||
*
|
||||
* @param data The data that will hold the copy of the data from disk
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
<T> T load(T data) throws IOException {
|
||||
return load(this.defaultKey, data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the saved data for the specified key.
|
||||
*
|
||||
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
<T> T load(String key, T data) throws IOException {
|
||||
ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
|
||||
|
||||
return load(wrap, data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the saved data for the specified key.
|
||||
*
|
||||
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
<T> T load(byte[] key, T data) throws IOException {
|
||||
return load(ByteArrayWrapper.wrap(key), data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the saved data for the specified key.
|
||||
*
|
||||
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||
*/
|
||||
@Override
|
||||
@SuppressWarnings("unchecked")
|
||||
public
|
||||
<T> T load(ByteArrayWrapper key, T data) throws IOException {
|
||||
Object source = get0(key);
|
||||
|
||||
if (source == null) {
|
||||
// returned was null, so we should take value as the default
|
||||
put(key, data);
|
||||
return data;
|
||||
}
|
||||
else {
|
||||
final Class<?> expectedClass = data.getClass();
|
||||
final Class<?> savedCLass = source.getClass();
|
||||
|
||||
if (!expectedClass.isAssignableFrom(savedCLass)) {
|
||||
throw new IOException("Saved value type '" + source.getClass() + "' is different that expected value");
|
||||
}
|
||||
}
|
||||
|
||||
return (T) source;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a object from pending or from storage
|
||||
*/
|
||||
private
|
||||
<T> T get0(ByteArrayWrapper key) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
// if the object in is pending, we get it from there
|
||||
try {
|
||||
this.actionLock.lock();
|
||||
|
||||
Object object = this.actionMap.get(key);
|
||||
|
||||
if (object != null) {
|
||||
@SuppressWarnings("unchecked")
|
||||
T returnObject = (T) object;
|
||||
return returnObject;
|
||||
}
|
||||
} finally {
|
||||
this.actionLock.unlock();
|
||||
}
|
||||
|
||||
// not found, so we have to go find it on disk
|
||||
return this.storage.get(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves the given data to storage with the associated key.
|
||||
* <p/>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
void put(String key, Object object) {
|
||||
put(ByteArrayWrapper.wrap(key), object);
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves the given data to storage with the associated key.
|
||||
* <p/>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
void put(byte[] key, Object object) {
|
||||
put(ByteArrayWrapper.wrap(key), object);
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves the given data to storage with the associated key.
|
||||
* <p/>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
void put(ByteArrayWrapper key, Object object) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
action(key, object);
|
||||
|
||||
// timer action runs on TIMER thread, not this thread
|
||||
this.timer.delay(this.milliSeconds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds the given object to the storage using a default (blank) key, OR -- if it has been registered, using it's registered key
|
||||
* <p/>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
void put(Object object) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
action(this.defaultKey, object);
|
||||
|
||||
// timer action runs on TIMER thread, not this thread
|
||||
this.timer.delay(this.milliSeconds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes an object from storage. To ALSO remove from the cache, use unRegister(key)
|
||||
*
|
||||
* @return true if the delete was successful. False if there were problems deleting the data.
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
boolean delete(String key) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
|
||||
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
|
||||
return this.storage.delete(wrap);
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the database and file.
|
||||
*/
|
||||
void close() {
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
|
||||
// have to "close" it after we run the timer!
|
||||
this.isOpen.set(false);
|
||||
|
||||
this.storage.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the file that backs this storage
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
File getFile() {
|
||||
return this.storage.getFile();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the backing file size.
|
||||
*
|
||||
* @return -1 if there was an error
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
long getFileSize() {
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
|
||||
return this.storage.getFileSize();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return true if there are objects queued to be written?
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
boolean hasWriteWaiting() {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
return this.timer.isWaiting();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the delay in milliseconds this will wait after the last action to flush the data to the disk
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
long getSaveDelay() {
|
||||
return this.milliSeconds;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param milliSeconds milliseconds to wait
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
void setSaveDelay(long milliSeconds) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
this.milliSeconds = milliSeconds;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the version of data stored in the database
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
int getVersion() {
|
||||
return this.storage.getVersion();
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the version of data stored in the database
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
void setVersion(int version) {
|
||||
this.storage.setVersion(version);
|
||||
}
|
||||
|
||||
private
|
||||
void action(ByteArrayWrapper key, Object object) {
|
||||
try {
|
||||
this.actionLock.lock();
|
||||
|
||||
// push action to map
|
||||
this.actionMap.put(key, object);
|
||||
} finally {
|
||||
this.actionLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
void increaseReference() {
|
||||
this.references.incrementAndGet();
|
||||
}
|
||||
|
||||
/**
|
||||
* return true when this is the last reference
|
||||
*/
|
||||
boolean decrementReference() {
|
||||
return this.references.decrementAndGet() <= 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected
|
||||
Object clone() throws CloneNotSupportedException {
|
||||
return super.clone();
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the storage to disk, immediately.
|
||||
* <p/>
|
||||
* This will save the ALL of the pending save actions to the file
|
||||
*/
|
||||
@Override
|
||||
public final
|
||||
void commit() {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the storage to disk, immediately.
|
||||
* <p/>
|
||||
* This will save the ALL of the pending save actions to the file
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
void commit(final String key, final Object object) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
action(ByteArrayWrapper.wrap(key), object);
|
||||
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the storage to disk, immediately.
|
||||
* <p/>
|
||||
* This will save the ALL of the pending save actions to the file
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
void commit(final byte[] key, final Object object) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
action(ByteArrayWrapper.wrap(key), object);
|
||||
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the storage to disk, immediately.
|
||||
* <p/>
|
||||
* This will save the ALL of the pending save actions to the file
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
void commit(final ByteArrayWrapper key, final Object object) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
action(key, object);
|
||||
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
}
|
||||
}
|
177
Dorkbox-Util/src/dorkbox/util/storage/DiskStorageIfface.java
Normal file
177
Dorkbox-Util/src/dorkbox/util/storage/DiskStorageIfface.java
Normal file
|
@ -0,0 +1,177 @@
|
|||
package dorkbox.util.storage;
|
||||
|
||||
import dorkbox.util.bytes.ByteArrayWrapper;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public
|
||||
interface DiskStorageIfface {
|
||||
/**
|
||||
* Returns the number of objects in the database.
|
||||
*/
|
||||
int size();
|
||||
|
||||
/**
|
||||
* Checks if there is a object corresponding to the given key.
|
||||
*/
|
||||
boolean contains(String key);
|
||||
|
||||
/**
|
||||
* Reads a object using the default (blank) key, and casts it to the expected class
|
||||
*/
|
||||
<T> T get();
|
||||
|
||||
/**
|
||||
* Reads a object using the specific key, and casts it to the expected class
|
||||
*/
|
||||
<T> T get(String key);
|
||||
|
||||
/**
|
||||
* Reads a object using the specific key, and casts it to the expected class
|
||||
*/
|
||||
<T> T get(byte[] key);
|
||||
|
||||
/**
|
||||
* Reads a object using the specific key, and casts it to the expected class
|
||||
*/
|
||||
<T> T get(ByteArrayWrapper key);
|
||||
|
||||
/**
|
||||
* Uses the DEFAULT key ("") to return saved data.
|
||||
* <p/>
|
||||
* This will check to see if there is an associated key for that data, if not - it will use data as the default
|
||||
*
|
||||
* @param data The data that will hold the copy of the data from disk
|
||||
*/
|
||||
<T> T load(T data) throws IOException;
|
||||
|
||||
/**
|
||||
* Returns the saved data for the specified key.
|
||||
*
|
||||
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||
*/
|
||||
<T> T load(String key, T data) throws IOException;
|
||||
|
||||
/**
|
||||
* Returns the saved data for the specified key.
|
||||
*
|
||||
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||
*/
|
||||
<T> T load(byte[] key, T data) throws IOException;
|
||||
|
||||
/**
|
||||
* Returns the saved data for the specified key.
|
||||
*
|
||||
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
<T> T load(ByteArrayWrapper key, T data) throws IOException;
|
||||
|
||||
/**
|
||||
* Saves the given data to storage with the associated key.
|
||||
* <p/>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
void put(String key, Object data);
|
||||
|
||||
/**
|
||||
* Saves the given data to storage with the associated key.
|
||||
* <p/>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
void put(byte[] key, Object data);
|
||||
|
||||
/**
|
||||
* Saves the given data to storage with the associated key.
|
||||
* <p/>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
void put(ByteArrayWrapper key, Object data);
|
||||
|
||||
/**
|
||||
* Adds the given object to the storage using a default (blank) key, OR -- if it has been registered, using it's registered key
|
||||
* <p/>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
void put(Object data);
|
||||
|
||||
/**
|
||||
* Deletes an object from storage. To ALSO remove from the cache, use unRegister(key)
|
||||
*
|
||||
* @return true if the delete was successful. False if there were problems deleting the data.
|
||||
*/
|
||||
boolean delete(String key);
|
||||
|
||||
/**
|
||||
* @return the file that backs this storage
|
||||
*/
|
||||
File getFile();
|
||||
|
||||
/**
|
||||
* Gets the backing file size.
|
||||
*
|
||||
* @return -1 if there was an error
|
||||
*/
|
||||
long getFileSize();
|
||||
|
||||
/**
|
||||
* @return true if there are objects queued to be written?
|
||||
*/
|
||||
boolean hasWriteWaiting();
|
||||
|
||||
/**
|
||||
* @return the delay in milliseconds this will wait after the last action to flush the data to the disk
|
||||
*/
|
||||
long getSaveDelay();
|
||||
|
||||
/**
|
||||
* @param milliSeconds milliseconds to wait
|
||||
*/
|
||||
void setSaveDelay(long milliSeconds);
|
||||
|
||||
/**
|
||||
* @return the version of data stored in the database
|
||||
*/
|
||||
int getVersion();
|
||||
|
||||
/**
|
||||
* Sets the version of data stored in the database
|
||||
*/
|
||||
void setVersion(int version);
|
||||
|
||||
/**
|
||||
* Save the storage to disk, immediately.
|
||||
* <p/>
|
||||
* This will save the ALL of the pending save actions to the file
|
||||
*/
|
||||
void commit();
|
||||
|
||||
/**
|
||||
* Save the storage to disk, immediately.
|
||||
* <p/>
|
||||
* This will save the ALL of the pending save actions to the file
|
||||
*/
|
||||
void commit(String key, Object object);
|
||||
|
||||
/**
|
||||
* Save the storage to disk, immediately.
|
||||
* <p/>
|
||||
* This will save the ALL of the pending save actions to the file
|
||||
*/
|
||||
void commit(byte[] key, Object object);
|
||||
|
||||
/**
|
||||
* Save the storage to disk, immediately.
|
||||
* <p/>
|
||||
* This will save the ALL of the pending save actions to the file
|
||||
*/
|
||||
void commit(ByteArrayWrapper key, Object object);
|
||||
}
|
176
Dorkbox-Util/src/dorkbox/util/storage/MakeStorage.java
Normal file
176
Dorkbox-Util/src/dorkbox/util/storage/MakeStorage.java
Normal file
|
@ -0,0 +1,176 @@
|
|||
package dorkbox.util.storage;
|
||||
|
||||
import dorkbox.util.SerializationManager;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public
|
||||
class MakeStorage {
|
||||
private static final Logger logger = LoggerFactory.getLogger(DiskStorage.class);
|
||||
@SuppressWarnings("SpellCheckingInspection")
|
||||
private static final Map<File, DiskStorageIfface> storages = new HashMap<File, DiskStorageIfface>(1);
|
||||
|
||||
public static
|
||||
DiskMaker Disk() {
|
||||
return new DiskMaker();
|
||||
}
|
||||
|
||||
public static
|
||||
MemoryMaker Memory() {
|
||||
return new MemoryMaker();
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the storage.
|
||||
*/
|
||||
public static
|
||||
void close(File file) {
|
||||
synchronized (storages) {
|
||||
DiskStorageIfface storage = storages.get(file);
|
||||
if (storage != null) {
|
||||
if (storage instanceof DiskStorage) {
|
||||
final DiskStorage diskStorage = (DiskStorage) storage;
|
||||
boolean isLastOne = diskStorage.decrementReference();
|
||||
if (isLastOne) {
|
||||
diskStorage.close();
|
||||
storages.remove(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the storage.
|
||||
*/
|
||||
public static
|
||||
void close(DiskStorageIfface _storage) {
|
||||
synchronized (storages) {
|
||||
File file = _storage.getFile();
|
||||
DiskStorageIfface storage = storages.get(file);
|
||||
if (storage != null) {
|
||||
if (storage instanceof DiskStorage) {
|
||||
final DiskStorage diskStorage = (DiskStorage) storage;
|
||||
boolean isLastOne = diskStorage.decrementReference();
|
||||
if (isLastOne) {
|
||||
diskStorage.close();
|
||||
storages.remove(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static
|
||||
void shutdown() {
|
||||
synchronized (storages) {
|
||||
Collection<DiskStorageIfface> values = storages.values();
|
||||
for (DiskStorageIfface storage : values) {
|
||||
if (storage instanceof DiskStorage) {
|
||||
//noinspection StatementWithEmptyBody
|
||||
final DiskStorage diskStorage = (DiskStorage) storage;
|
||||
while (!diskStorage.decrementReference()) {
|
||||
}
|
||||
diskStorage.close();
|
||||
}
|
||||
}
|
||||
storages.clear();
|
||||
}
|
||||
}
|
||||
|
||||
public static
|
||||
void delete(File file) {
|
||||
synchronized (storages) {
|
||||
DiskStorageIfface remove = storages.remove(file);
|
||||
if (remove != null && remove instanceof DiskStorage) {
|
||||
((DiskStorage) remove).close();
|
||||
}
|
||||
//noinspection ResultOfMethodCallIgnored
|
||||
file.delete();
|
||||
}
|
||||
}
|
||||
|
||||
public static
|
||||
void delete(DiskStorageIfface storage) {
|
||||
File file = storage.getFile();
|
||||
delete(file);
|
||||
}
|
||||
|
||||
|
||||
public static
|
||||
class DiskMaker {
|
||||
private File file;
|
||||
private SerializationManager serializationManager;
|
||||
|
||||
public
|
||||
DiskMaker file(File file) {
|
||||
this.file = file;
|
||||
return this;
|
||||
}
|
||||
|
||||
public
|
||||
DiskMaker serializer(SerializationManager serializationManager) {
|
||||
this.serializationManager = serializationManager;
|
||||
return this;
|
||||
}
|
||||
|
||||
public
|
||||
DiskStorageIfface make() {
|
||||
if (file == null) {
|
||||
throw new IllegalArgumentException("file cannot be null!");
|
||||
}
|
||||
|
||||
// if we load from a NEW storage at the same location as an ALREADY EXISTING storage,
|
||||
// without saving the existing storage first --- whoops!
|
||||
synchronized (storages) {
|
||||
DiskStorageIfface storage = storages.get(file);
|
||||
|
||||
if (storage != null) {
|
||||
if (storage instanceof DiskStorage) {
|
||||
boolean waiting = storage.hasWriteWaiting();
|
||||
// we want this storage to be in a fresh state
|
||||
if (waiting) {
|
||||
storage.commit();
|
||||
}
|
||||
((DiskStorage) storage).increaseReference();
|
||||
}
|
||||
else {
|
||||
throw new RuntimeException("Unable to change storage types for: " + file);
|
||||
}
|
||||
}
|
||||
else {
|
||||
try {
|
||||
storage = new DiskStorage(file, serializationManager);
|
||||
storages.put(file, storage);
|
||||
} catch (IOException e) {
|
||||
logger.error("Unable to open storage", e);
|
||||
}
|
||||
}
|
||||
|
||||
return storage;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public static
|
||||
class MemoryMaker {
|
||||
private SerializationManager serializationManager;
|
||||
|
||||
public
|
||||
MemoryMaker serializer(SerializationManager serializationManager) {
|
||||
this.serializationManager = serializationManager;
|
||||
return this;
|
||||
}
|
||||
|
||||
MemoryStorage make() {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
253
Dorkbox-Util/src/dorkbox/util/storage/MemoryStorage.java
Normal file
253
Dorkbox-Util/src/dorkbox/util/storage/MemoryStorage.java
Normal file
|
@ -0,0 +1,253 @@
|
|||
package dorkbox.util.storage;
|
||||
|
||||
import dorkbox.util.bytes.ByteArrayWrapper;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
public
|
||||
class MemoryStorage implements DiskStorageIfface {
|
||||
private final ConcurrentHashMap<ByteArrayWrapper, Object> storage;
|
||||
private final ByteArrayWrapper defaultKey;
|
||||
private int version;
|
||||
|
||||
private
|
||||
MemoryStorage() throws IOException {
|
||||
this.storage = new ConcurrentHashMap<ByteArrayWrapper, Object>();
|
||||
this.defaultKey = ByteArrayWrapper.wrap("");
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns the number of objects in the database.
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
int size() {
|
||||
return storage.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if there is a object corresponding to the given key.
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
boolean contains(final String key) {
|
||||
return storage.containsKey(ByteArrayWrapper.wrap(key));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a object using the default (blank) key, and casts it to the expected class
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public
|
||||
<T> T get() {
|
||||
return (T) storage.get(defaultKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a object using the specific key, and casts it to the expected class
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
<T> T get(final String key) {
|
||||
return get(ByteArrayWrapper.wrap(key));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a object using the specific key, and casts it to the expected class
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
<T> T get(final byte[] key) {
|
||||
return get(ByteArrayWrapper.wrap(key));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a object using the specific key, and casts it to the expected class
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public
|
||||
<T> T get(final ByteArrayWrapper key) {
|
||||
return (T) storage.get(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Uses the DEFAULT key ("") to return saved data.
|
||||
* <p/>
|
||||
* This will check to see if there is an associated key for that data, if not - it will use data as the default
|
||||
*
|
||||
* @param data The data that will hold the copy of the data from disk
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
<T> T load(T data) throws IOException {
|
||||
return load(this.defaultKey, data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the saved data for the specified key.
|
||||
*
|
||||
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
<T> T load(String key, T data) throws IOException {
|
||||
ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
|
||||
|
||||
return load(wrap, data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the saved data for the specified key.
|
||||
*
|
||||
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
<T> T load(byte[] key, T data) throws IOException {
|
||||
return load(ByteArrayWrapper.wrap(key), data);
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@Override
|
||||
public
|
||||
<T> T load(final ByteArrayWrapper key, final T data) throws IOException {
|
||||
final Object o = storage.get(key);
|
||||
if (o == null) {
|
||||
storage.put(key, data);
|
||||
}
|
||||
return (T) o;
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves the given data to storage with the associated key.
|
||||
* <p/>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
void put(final String key, final Object data) {
|
||||
put(ByteArrayWrapper.wrap(key), data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves the given data to storage with the associated key.
|
||||
* <p/>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
void put(final byte[] key, final Object data) {
|
||||
put(ByteArrayWrapper.wrap(key), data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves the given data to storage with the associated key.
|
||||
* <p/>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
void put(final ByteArrayWrapper key, final Object object) {
|
||||
storage.put(key, object);
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves the given data to storage with the associated key.
|
||||
* <p/>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
void put(final Object data) {
|
||||
put(defaultKey, data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes an object from storage. To ALSO remove from the cache, use unRegister(key)
|
||||
*
|
||||
* @return true if the delete was successful. False if there were problems deleting the data.
|
||||
*/
|
||||
@Override
|
||||
public
|
||||
boolean delete(final String key) {
|
||||
storage.remove(ByteArrayWrapper.wrap(key));
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
File getFile() {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
long getFileSize() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
boolean hasWriteWaiting() {
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
long getSaveDelay() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
void setSaveDelay(final long milliSeconds) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized
|
||||
int getVersion() {
|
||||
return version;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized
|
||||
void setVersion(final int version) {
|
||||
this.version = version;
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
void commit() {
|
||||
// no-op
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
void commit(final String key, final Object object) {
|
||||
// no-op
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
void commit(final byte[] key, final Object object) {
|
||||
// no-op
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
void commit(final ByteArrayWrapper key, final Object object) {
|
||||
// no-op
|
||||
}
|
||||
}
|
|
@ -15,6 +15,11 @@
|
|||
*/
|
||||
package dorkbox.util.storage;
|
||||
|
||||
import com.esotericsoftware.kryo.io.Input;
|
||||
import com.esotericsoftware.kryo.io.Output;
|
||||
import dorkbox.util.SerializationManager;
|
||||
import dorkbox.util.bytes.ByteArrayWrapper;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
|
@ -24,13 +29,9 @@ import java.nio.channels.FileLock;
|
|||
import java.util.zip.DeflaterOutputStream;
|
||||
import java.util.zip.InflaterInputStream;
|
||||
|
||||
import com.esotericsoftware.kryo.Kryo;
|
||||
import com.esotericsoftware.kryo.io.Input;
|
||||
import com.esotericsoftware.kryo.io.Output;
|
||||
|
||||
import dorkbox.util.bytes.ByteArrayWrapper;
|
||||
|
||||
public class Metadata {
|
||||
@SuppressWarnings("unused")
|
||||
public
|
||||
class Metadata {
|
||||
// The length of a key in the index.
|
||||
// SHA256 is 32 bytes long.
|
||||
private static final int KEY_SIZE = 32;
|
||||
|
@ -78,7 +79,8 @@ public class Metadata {
|
|||
/**
|
||||
* Returns a file pointer in the index pointing to the first byte in the KEY located at the given index position.
|
||||
*/
|
||||
static final long getMetaDataPointer(int position) {
|
||||
static
|
||||
long getMetaDataPointer(int position) {
|
||||
return StorageBase.FILE_HEADERS_REGION_LENGTH + INDEX_ENTRY_LENGTH * position;
|
||||
}
|
||||
|
||||
|
@ -86,17 +88,20 @@ public class Metadata {
|
|||
* Returns a file pointer in the index pointing to the first byte in the RECORD pointer located at the given index
|
||||
* position.
|
||||
*/
|
||||
static final long getDataPointer(int position) {
|
||||
long l = Metadata.getMetaDataPointer(position) + KEY_SIZE;
|
||||
return l;
|
||||
static
|
||||
long getDataPointer(int position) {
|
||||
return Metadata.getMetaDataPointer(position) + KEY_SIZE;
|
||||
}
|
||||
|
||||
|
||||
private Metadata(ByteArrayWrapper key) {
|
||||
private
|
||||
Metadata(ByteArrayWrapper key) {
|
||||
this.key = key;
|
||||
}
|
||||
|
||||
/** we don't know how much data there is until AFTER we write the data */
|
||||
/**
|
||||
* we don't know how much data there is until AFTER we write the data
|
||||
*/
|
||||
Metadata(ByteArrayWrapper key, int recordIndex, long dataPointer) {
|
||||
this(key, recordIndex, dataPointer, 0);
|
||||
}
|
||||
|
@ -123,11 +128,13 @@ public class Metadata {
|
|||
/**
|
||||
* Reads the ith HEADER (key + metadata) from the index.
|
||||
*/
|
||||
static Metadata readHeader(RandomAccessFile file, int position) throws IOException {
|
||||
static
|
||||
Metadata readHeader(RandomAccessFile file, int position) throws IOException {
|
||||
byte[] buf = new byte[KEY_SIZE];
|
||||
long origHeaderKeyPointer = Metadata.getMetaDataPointer(position);
|
||||
|
||||
FileLock lock = file.getChannel().lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true);
|
||||
FileLock lock = file.getChannel()
|
||||
.lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true);
|
||||
file.seek(origHeaderKeyPointer);
|
||||
file.readFully(buf);
|
||||
lock.release();
|
||||
|
@ -136,7 +143,8 @@ public class Metadata {
|
|||
r.indexPosition = position;
|
||||
|
||||
long recordHeaderPointer = Metadata.getDataPointer(position);
|
||||
lock = file.getChannel().lock(origHeaderKeyPointer, KEY_SIZE, true);
|
||||
lock = file.getChannel()
|
||||
.lock(origHeaderKeyPointer, KEY_SIZE, true);
|
||||
file.seek(recordHeaderPointer);
|
||||
r.dataPointer = file.readLong();
|
||||
r.dataCapacity = file.readInt();
|
||||
|
@ -153,7 +161,8 @@ public class Metadata {
|
|||
void writeMetaDataInfo(RandomAccessFile file) throws IOException {
|
||||
long recordKeyPointer = Metadata.getMetaDataPointer(this.indexPosition);
|
||||
|
||||
FileLock lock = file.getChannel().lock(recordKeyPointer, KEY_SIZE, false);
|
||||
FileLock lock = file.getChannel()
|
||||
.lock(recordKeyPointer, KEY_SIZE, false);
|
||||
file.seek(recordKeyPointer);
|
||||
file.write(this.key.getBytes());
|
||||
lock.release();
|
||||
|
@ -162,7 +171,8 @@ public class Metadata {
|
|||
void writeDataInfo(RandomAccessFile file) throws IOException {
|
||||
long recordHeaderPointer = getDataPointer(this.indexPosition);
|
||||
|
||||
FileLock lock = file.getChannel().lock(recordHeaderPointer, POINTER_INFO_SIZE, false);
|
||||
FileLock lock = file.getChannel()
|
||||
.lock(recordHeaderPointer, POINTER_INFO_SIZE, false);
|
||||
file.seek(recordHeaderPointer);
|
||||
file.writeLong(this.dataPointer);
|
||||
file.writeInt(this.dataCapacity);
|
||||
|
@ -177,13 +187,17 @@ public class Metadata {
|
|||
byte[] buf = new byte[KEY_SIZE];
|
||||
|
||||
long origHeaderKeyPointer = Metadata.getMetaDataPointer(this.indexPosition);
|
||||
FileLock lock = file.getChannel().lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true);
|
||||
FileLock lock = file.getChannel()
|
||||
.lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true);
|
||||
|
||||
file.seek(origHeaderKeyPointer);
|
||||
file.readFully(buf);
|
||||
lock.release();
|
||||
|
||||
long newHeaderKeyPointer = Metadata.getMetaDataPointer(newIndex);
|
||||
lock = file.getChannel().lock(newHeaderKeyPointer, INDEX_ENTRY_LENGTH, false);
|
||||
lock = file.getChannel()
|
||||
.lock(newHeaderKeyPointer, INDEX_ENTRY_LENGTH, false);
|
||||
|
||||
file.seek(newHeaderKeyPointer);
|
||||
file.write(buf);
|
||||
lock.release();
|
||||
|
@ -205,7 +219,8 @@ public class Metadata {
|
|||
this.dataPointer = position;
|
||||
this.dataCapacity = this.dataCount;
|
||||
|
||||
FileLock lock = file.getChannel().lock(position, this.dataCount, false);
|
||||
FileLock lock = file.getChannel()
|
||||
.lock(position, this.dataCount, false);
|
||||
|
||||
// update the file size
|
||||
file.setLength(position + this.dataCount);
|
||||
|
@ -230,7 +245,8 @@ public class Metadata {
|
|||
byte[] buf = new byte[this.dataCount];
|
||||
// System.err.print("Reading data: " + this.indexPosition + " @ " + this.dataPointer + "-" + (this.dataPointer+this.dataCount) + " -- ");
|
||||
|
||||
FileLock lock = file.getChannel().lock(this.dataPointer, this.dataCount, true);
|
||||
FileLock lock = file.getChannel()
|
||||
.lock(this.dataPointer, this.dataCount, true);
|
||||
file.seek(this.dataPointer);
|
||||
file.readFully(buf);
|
||||
lock.release();
|
||||
|
@ -243,20 +259,25 @@ public class Metadata {
|
|||
* Reads the record data for the given record header.
|
||||
*/
|
||||
|
||||
<T> T readData(Kryo kryo, InflaterInputStream inputStream) {
|
||||
static
|
||||
<T> T readData(SerializationManager serializationManager, InflaterInputStream inputStream) {
|
||||
Input input = new Input(inputStream, 1024); // read 1024 at a time
|
||||
@SuppressWarnings("unchecked")
|
||||
T readObject = (T) kryo.readClassAndObject(input);
|
||||
T readObject = (T) serializationManager.readClassAndObject(input);
|
||||
return readObject;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes data to the end of the file (which is where the datapointer is at)
|
||||
*/
|
||||
void writeDataFast(Kryo kryo, Object data, DeflaterOutputStream outputStream) throws IOException {
|
||||
static
|
||||
void writeDataFast(final SerializationManager serializationManager,
|
||||
final Object data,
|
||||
final RandomAccessFile file,
|
||||
final DeflaterOutputStream outputStream) throws IOException {
|
||||
// HAVE TO LOCK BEFORE THIS IS CALLED! (AND FREE AFTERWARDS!)
|
||||
Output output = new Output(outputStream, 1024); // write 1024 at a time
|
||||
kryo.writeClassAndObject(output, data);
|
||||
serializationManager.writeClassAndObject(output, data);
|
||||
output.flush();
|
||||
|
||||
outputStream.flush(); // sync-flush is enabled, so the output stream will finish compressing data.
|
||||
|
@ -265,7 +286,8 @@ public class Metadata {
|
|||
void writeData(ByteArrayOutputStream byteArrayOutputStream, RandomAccessFile file) throws IOException {
|
||||
this.dataCount = byteArrayOutputStream.size();
|
||||
|
||||
FileLock lock = file.getChannel().lock(this.dataPointer, this.dataCount, false);
|
||||
FileLock lock = file.getChannel()
|
||||
.lock(this.dataPointer, this.dataCount, false);
|
||||
|
||||
FileOutputStream out = new FileOutputStream(file.getFD());
|
||||
file.seek(this.dataPointer);
|
||||
|
@ -275,8 +297,9 @@ public class Metadata {
|
|||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "RecordHeader [dataPointer=" + this.dataPointer + ", dataCount=" + this.dataCount + ", dataCapacity="
|
||||
+ this.dataCapacity + ", indexPosition=" + this.indexPosition + "]";
|
||||
public
|
||||
String toString() {
|
||||
return "RecordHeader [dataPointer=" + this.dataPointer + ", dataCount=" + this.dataCount + ", dataCapacity=" + this.dataCapacity +
|
||||
", indexPosition=" + this.indexPosition + "]";
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,568 +0,0 @@
|
|||
/*
|
||||
* Copyright 2014 dorkbox, llc
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package dorkbox.util.storage;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
import org.bouncycastle.crypto.digests.SHA256Digest;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import dorkbox.util.DelayTimer;
|
||||
import dorkbox.util.OS;
|
||||
import dorkbox.util.bytes.ByteArrayWrapper;
|
||||
|
||||
/**
|
||||
* Nothing spectacular about this storage -- it allows for persistent storage of objects to disk.
|
||||
*/
|
||||
public class Storage {
|
||||
private static final Logger logger = LoggerFactory.getLogger(Storage.class);
|
||||
|
||||
private static Map<File, Storage> storages = new HashMap<File, Storage>(1);
|
||||
|
||||
public static Storage open(String file) {
|
||||
if (file == null || file.isEmpty()) {
|
||||
throw new IllegalArgumentException("file cannot be null or empty!");
|
||||
}
|
||||
return open(new File(file));
|
||||
}
|
||||
|
||||
/**
|
||||
* Two types of storage.
|
||||
* Raw) save/load a single object to disk (better for really large files)
|
||||
* Normal) save/load key/value objects to disk (better for multiple types of data in a single file)
|
||||
*/
|
||||
public static Storage open(File file) {
|
||||
if (file == null) {
|
||||
throw new IllegalArgumentException("file cannot be null!");
|
||||
}
|
||||
|
||||
// if we load from a NEW storage at the same location as an ALREADY EXISTING storage,
|
||||
// without saving the existing storage first --- whoops!
|
||||
synchronized (storages) {
|
||||
Storage storage = storages.get(file);
|
||||
|
||||
if (storage != null) {
|
||||
boolean waiting = storage.hasWriteWaiting();
|
||||
// we want this storage to be in a fresh state
|
||||
if (waiting) {
|
||||
storage.saveNow();
|
||||
}
|
||||
storage.increaseReference();
|
||||
} else {
|
||||
try {
|
||||
storage = new Storage(file);
|
||||
storages.put(file, storage);
|
||||
} catch (IOException e) {
|
||||
logger.error("Unable to open storage", e);
|
||||
}
|
||||
}
|
||||
|
||||
return storage;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Closes the storage.
|
||||
*/
|
||||
public static void close(File file) {
|
||||
synchronized (storages) {
|
||||
Storage storage = storages.get(file);
|
||||
if (storage != null) {
|
||||
boolean isLastOne = storage.decrementReference();
|
||||
if (isLastOne) {
|
||||
storage.close();
|
||||
storages.remove(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the storage.
|
||||
*/
|
||||
public static void close(Storage _storage) {
|
||||
synchronized (storages) {
|
||||
File file = _storage.getFile();
|
||||
Storage storage = storages.get(file);
|
||||
if (storage != null) {
|
||||
boolean isLastOne = storage.decrementReference();
|
||||
if (isLastOne) {
|
||||
storage.close();
|
||||
storages.remove(file);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static void shutdown() {
|
||||
synchronized(storages) {
|
||||
Collection<Storage> values = storages.values();
|
||||
for (Storage storage : values) {
|
||||
while (!storage.decrementReference()) {
|
||||
}
|
||||
storage.close();
|
||||
}
|
||||
storages.clear();
|
||||
}
|
||||
}
|
||||
|
||||
public static void delete(File file) {
|
||||
synchronized(storages) {
|
||||
Storage remove = storages.remove(file);
|
||||
if (remove != null) {
|
||||
remove.close();
|
||||
}
|
||||
file.delete();
|
||||
}
|
||||
}
|
||||
|
||||
public static void delete(Storage storage) {
|
||||
File file = storage.getFile();
|
||||
delete(file);
|
||||
}
|
||||
|
||||
|
||||
private static void copyFields(Object source, Object dest) {
|
||||
Class<? extends Object> sourceClass = source.getClass();
|
||||
Class<? extends Object> destClass = dest.getClass();
|
||||
|
||||
if (sourceClass != destClass) {
|
||||
throw new IllegalArgumentException("Source and Dest objects are not of the same class!");
|
||||
}
|
||||
|
||||
// have to walk up the object hierarchy.
|
||||
while (destClass != Object.class) {
|
||||
Field[] destFields = destClass.getDeclaredFields();
|
||||
|
||||
for (Field destField : destFields) {
|
||||
String name = destField.getName();
|
||||
try {
|
||||
Field sourceField = sourceClass.getDeclaredField(name);
|
||||
destField.setAccessible(true);
|
||||
sourceField.setAccessible(true);
|
||||
|
||||
Object sourceObj = sourceField.get(source);
|
||||
|
||||
if (sourceObj instanceof Map) {
|
||||
Object destObj = destField.get(dest);
|
||||
if (destObj == null) {
|
||||
destField.set(dest, sourceObj);
|
||||
} else if (destObj instanceof Map) {
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<Object, Object> sourceMap = (Map<Object, Object>) sourceObj;
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<Object, Object> destMap = (Map<Object, Object>) destObj;
|
||||
|
||||
destMap.clear();
|
||||
Iterator<?> entries = sourceMap.entrySet().iterator();
|
||||
while (entries.hasNext()) {
|
||||
Map.Entry<?, ?> entry = (Map.Entry<?, ?>)entries.next();
|
||||
Object key = entry.getKey();
|
||||
Object value = entry.getValue();
|
||||
destMap.put(key, value);
|
||||
}
|
||||
|
||||
} else {
|
||||
logger.error("Incompatible field type! '{}'", name);
|
||||
}
|
||||
} else {
|
||||
destField.set(dest, sourceObj);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("Unable to copy field: {}", name, e);
|
||||
}
|
||||
}
|
||||
|
||||
destClass = destClass.getSuperclass();
|
||||
sourceClass = sourceClass.getSuperclass();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
private volatile long milliSeconds = 3000L;
|
||||
private volatile DelayTimer timer;
|
||||
|
||||
private final ByteArrayWrapper defaultKey;
|
||||
private final StorageBase storage;
|
||||
|
||||
private AtomicInteger references = new AtomicInteger(1);
|
||||
|
||||
private final ReentrantLock actionLock = new ReentrantLock();
|
||||
private volatile Map<ByteArrayWrapper, Object> actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
|
||||
|
||||
private AtomicBoolean isOpen = new AtomicBoolean(false);
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Creates or opens a new database file.
|
||||
*/
|
||||
private Storage(File storageFile) throws IOException {
|
||||
this.storage = new StorageBase(storageFile);
|
||||
this.defaultKey = wrap("");
|
||||
|
||||
this.timer = new DelayTimer("Storage Writer", false, new DelayTimer.Callback() {
|
||||
@Override
|
||||
public void execute() {
|
||||
Map<ByteArrayWrapper, Object> actions = Storage.this.actionMap;
|
||||
|
||||
ReentrantLock actionLock2 = Storage.this.actionLock;
|
||||
|
||||
try {
|
||||
actionLock2.lock();
|
||||
|
||||
// do a fast swap on the actionMap.
|
||||
Storage.this.actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
|
||||
} finally {
|
||||
actionLock2.unlock();
|
||||
}
|
||||
|
||||
Storage.this.storage.doActionThings(actions);
|
||||
}
|
||||
});
|
||||
|
||||
this.isOpen.set(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the number of objects in the database.
|
||||
* <p>
|
||||
* SLOW because this must save all data to disk first!
|
||||
*/
|
||||
public final int size() {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
// flush actions
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
|
||||
return this.storage.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if there is a object corresponding to the given key.
|
||||
*/
|
||||
public final boolean contains(String key) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
return this.storage.contains(wrap(key));
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a object using the default (blank) key, and casts it to the expected class
|
||||
*/
|
||||
public final <T> T get() {
|
||||
return get0(this.defaultKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a object using the specific key, and casts it to the expected class
|
||||
*/
|
||||
public final <T> T get(String key) {
|
||||
ByteArrayWrapper wrap = wrap(key);
|
||||
return get0(wrap);
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies the saved data (from) into the passed-in data. This just assigns all of the values from one to the other.
|
||||
* <p>
|
||||
* This will check to see if there is an associated key for that data, if not - it will use the default (blank)
|
||||
*
|
||||
* @param data The data that will hold the copy of the data from disk
|
||||
*/
|
||||
public void load(Object data) {
|
||||
Object source = get();
|
||||
|
||||
if (source != null) {
|
||||
Storage.copyFields(source, data);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Copies the saved data (from) into the passed-in data. This just assigns all of the values from one to the other.
|
||||
* <p>
|
||||
* The key/data will be associated for the lifetime of the object.
|
||||
*
|
||||
* @param object The object that will hold the copy of the data from disk (but not change the reference) once this is completed
|
||||
*/
|
||||
public void load(String key, Object object) {
|
||||
ByteArrayWrapper wrap = wrap(key);
|
||||
|
||||
Object source = get0(wrap);
|
||||
if (source != null) {
|
||||
Storage.copyFields(source, object);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a object from pending or from storage
|
||||
*/
|
||||
private final <T> T get0(ByteArrayWrapper key) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
// if the object in is pending, we get it from there
|
||||
try {
|
||||
this.actionLock.lock();
|
||||
|
||||
Object object = this.actionMap.get(key);
|
||||
|
||||
if (object != null) {
|
||||
@SuppressWarnings("unchecked")
|
||||
T returnObject = (T) object;
|
||||
return returnObject;
|
||||
}
|
||||
} finally {
|
||||
this.actionLock.unlock();
|
||||
}
|
||||
|
||||
// not found, so we have to go find it on disk
|
||||
return this.storage.get(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the storage to disk, once xxxx milli-seconds have passed.
|
||||
* This is to help prevent thrashing the disk, or wearing it out on multiple, rapid, changes.
|
||||
* <p>
|
||||
* This will save the ALL of the pending save actions to the file
|
||||
*/
|
||||
public final void saveNow() {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves the given data to storage with the associated key.
|
||||
* <p>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
public final void save(String key, Object object) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
ByteArrayWrapper wrap = wrap(key);
|
||||
action(wrap, object);
|
||||
|
||||
// timer action runs on TIMER thread, not this thread
|
||||
this.timer.delay(this.milliSeconds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves the given object to storage with the associated key.
|
||||
* <p>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
public final void saveNow(String key, Object object) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
ByteArrayWrapper wrap = wrap(key);
|
||||
action(wrap, object);
|
||||
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds the given object to the storage using a default (blank) key, OR -- if it has been registered, using it's registered key
|
||||
* <p>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
public final void save(Object object) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
action(this.defaultKey, object);
|
||||
|
||||
// timer action runs on TIMER thread, not this thread
|
||||
this.timer.delay(this.milliSeconds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds the given object to the storage using a default (blank) key
|
||||
* <p>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
public final void saveNow(Object object) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
action(this.defaultKey, object);
|
||||
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes an object from storage. To ALSO remove from the cache, use unRegister(key)
|
||||
*
|
||||
* @return true if the delete was successful. False if there were problems deleting the data.
|
||||
*/
|
||||
public final boolean delete(String key) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
ByteArrayWrapper wrap = wrap(key);
|
||||
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
|
||||
return this.storage.delete(wrap);
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the database and file.
|
||||
*/
|
||||
private final void close() {
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
|
||||
// have to "close" it after we run the timer!
|
||||
this.isOpen.set(false);
|
||||
|
||||
this.storage.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the file that backs this storage
|
||||
*/
|
||||
public final File getFile() {
|
||||
return this.storage.getFile();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the backing file size.
|
||||
*
|
||||
* @return -1 if there was an error
|
||||
*/
|
||||
public final long getFileSize() {
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
|
||||
return this.storage.getFileSize();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @return true if there are objects queued to be written?
|
||||
*/
|
||||
public final boolean hasWriteWaiting() {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
return this.timer.isWaiting();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param delay milliseconds to wait
|
||||
*/
|
||||
public final void setSaveDelay(long milliSeconds) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
this.milliSeconds = milliSeconds;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the delay in milliseconds this will wait after the last action to flush the data to the disk
|
||||
*/
|
||||
public final long getSaveDelay() {
|
||||
return this.milliSeconds;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return the version of data stored in the database
|
||||
*/
|
||||
public final int getVersion() {
|
||||
return this.storage.getVersion();
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the version of data stored in the database
|
||||
*/
|
||||
public final void setVersion(int version) {
|
||||
this.storage.setVersion(version);
|
||||
}
|
||||
|
||||
|
||||
private final ByteArrayWrapper wrap(String key) {
|
||||
byte[] bytes = key.getBytes(OS.UTF_8);
|
||||
|
||||
SHA256Digest digest = new SHA256Digest();
|
||||
digest.update(bytes, 0, bytes.length);
|
||||
byte[] hashBytes = new byte[digest.getDigestSize()];
|
||||
digest.doFinal(hashBytes, 0);
|
||||
ByteArrayWrapper wrap = ByteArrayWrapper.wrap(hashBytes);
|
||||
|
||||
return wrap;
|
||||
}
|
||||
|
||||
private void action(ByteArrayWrapper key, Object object) {
|
||||
try {
|
||||
this.actionLock.lock();
|
||||
|
||||
// push action to map
|
||||
this.actionMap.put(key, object);
|
||||
} finally {
|
||||
this.actionLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
private final void increaseReference() {
|
||||
this.references.incrementAndGet();
|
||||
}
|
||||
|
||||
/** return true when this is the last reference */
|
||||
private final boolean decrementReference() {
|
||||
return this.references.decrementAndGet() == 0;
|
||||
}
|
||||
}
|
|
@ -15,16 +15,16 @@
|
|||
*/
|
||||
package dorkbox.util.storage;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.RandomAccessFile;
|
||||
import com.esotericsoftware.kryo.KryoException;
|
||||
import com.esotericsoftware.kryo.io.Output;
|
||||
import dorkbox.util.SerializationManager;
|
||||
import dorkbox.util.bytes.ByteArrayWrapper;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.*;
|
||||
import java.lang.ref.WeakReference;
|
||||
import java.nio.channels.FileLock;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
|
@ -35,20 +35,15 @@ import java.util.zip.DeflaterOutputStream;
|
|||
import java.util.zip.Inflater;
|
||||
import java.util.zip.InflaterInputStream;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.esotericsoftware.kryo.Kryo;
|
||||
import com.esotericsoftware.kryo.KryoException;
|
||||
import com.esotericsoftware.kryo.io.Output;
|
||||
|
||||
import dorkbox.util.bytes.ByteArrayWrapper;
|
||||
|
||||
// a note on file locking between c and java
|
||||
// http://panks-dev.blogspot.de/2008/04/linux-file-locks-java-and-others.html
|
||||
// Also, file locks on linux are ADVISORY. if an app doesn't care about locks, then it can do stuff -- even if locked by another app
|
||||
|
||||
public class StorageBase {
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
public
|
||||
class StorageBase {
|
||||
private final Logger logger = LoggerFactory.getLogger(getClass());
|
||||
|
||||
|
||||
|
@ -69,11 +64,10 @@ public class StorageBase {
|
|||
private final Map<ByteArrayWrapper, Metadata> memoryIndex;
|
||||
|
||||
// determines how much the index will grow by
|
||||
private Float weight;
|
||||
private final Float weight;
|
||||
|
||||
// The keys are weak! When they go, the map entry is removed!
|
||||
private final ReentrantLock referenceLock = new ReentrantLock();
|
||||
private volatile Map<ByteArrayWrapper, Object> referencePendingWrite = new HashMap<ByteArrayWrapper, Object>();
|
||||
|
||||
|
||||
private final File baseFile;
|
||||
|
@ -97,32 +91,29 @@ public class StorageBase {
|
|||
|
||||
|
||||
// save references to these, so they don't have to be created/destroyed any time there is I/O
|
||||
private final Kryo kryo;
|
||||
private final SerializationManager serializationManager;
|
||||
|
||||
private Deflater deflater;
|
||||
private DeflaterOutputStream outputStream;
|
||||
private final Deflater deflater;
|
||||
private final DeflaterOutputStream outputStream;
|
||||
|
||||
private Inflater inflater;
|
||||
private InflaterInputStream inputStream;
|
||||
private final Inflater inflater;
|
||||
private final InflaterInputStream inputStream;
|
||||
|
||||
/**
|
||||
* Creates or opens a new database file.
|
||||
*/
|
||||
StorageBase(String filePath) throws IOException {
|
||||
this(new File(filePath));
|
||||
}
|
||||
StorageBase(File filePath, SerializationManager serializationManager) throws IOException {
|
||||
this.serializationManager = serializationManager;
|
||||
|
||||
/**
|
||||
* Creates or opens a new database file.
|
||||
*/
|
||||
StorageBase(File filePath) throws IOException {
|
||||
this.logger.info("Opening storage file: '{}'", filePath.getAbsolutePath());
|
||||
|
||||
this.baseFile = filePath;
|
||||
|
||||
File parentFile = this.baseFile.getParentFile();
|
||||
if (parentFile != null) {
|
||||
parentFile.mkdirs();
|
||||
if (parentFile != null && !parentFile.exists()) {
|
||||
if (!parentFile.mkdirs()) {
|
||||
throw new IOException("Unable to create dirs for: " + filePath);
|
||||
}
|
||||
}
|
||||
|
||||
this.file = new RandomAccessFile(this.baseFile, "rw");
|
||||
|
@ -132,7 +123,8 @@ public class StorageBase {
|
|||
this.databaseVersion = this.file.readInt();
|
||||
this.numberOfRecords = this.file.readInt();
|
||||
this.dataPosition = this.file.readLong();
|
||||
} else {
|
||||
}
|
||||
else {
|
||||
setVersionNumber(this.file, 0);
|
||||
|
||||
// always start off with 4 records
|
||||
|
@ -149,18 +141,16 @@ public class StorageBase {
|
|||
throw new IllegalArgumentException("Unable to parse header information from storage. Maybe it's corrupted?");
|
||||
}
|
||||
|
||||
//noinspection AutoBoxing
|
||||
this.logger.info("Storage version: {}", this.databaseVersion);
|
||||
|
||||
|
||||
this.kryo = new Kryo();
|
||||
this.kryo.setRegistrationRequired(false);
|
||||
|
||||
this.deflater = new Deflater(7, true);
|
||||
this.outputStream = new DeflaterOutputStream(new FileOutputStream(this.file.getFD()), this.deflater, 65536, true);
|
||||
|
||||
this.inflater = new Inflater(true);
|
||||
this.inputStream = new InflaterInputStream(new FileInputStream(this.file.getFD()), this.inflater, 65536);
|
||||
|
||||
//noinspection AutoBoxing
|
||||
this.weight = .5F;
|
||||
this.memoryIndex = new ConcurrentHashMap<ByteArrayWrapper, Metadata>(this.numberOfRecords);
|
||||
|
||||
|
@ -183,7 +173,8 @@ public class StorageBase {
|
|||
/**
|
||||
* Returns the current number of records in the database.
|
||||
*/
|
||||
final int size() {
|
||||
final
|
||||
int size() {
|
||||
// wrapper flushes first (protected by lock)
|
||||
// not protected by lock
|
||||
return this.memoryIndex.size();
|
||||
|
@ -192,21 +183,19 @@ public class StorageBase {
|
|||
/**
|
||||
* Checks if there is a record belonging to the given key.
|
||||
*/
|
||||
final boolean contains(ByteArrayWrapper key) {
|
||||
final
|
||||
boolean contains(ByteArrayWrapper key) {
|
||||
// protected by lock
|
||||
|
||||
// check to see if it's in the pending ops
|
||||
if (this.referencePendingWrite.containsKey(key)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return this.memoryIndex.containsKey(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return an object for a specified key ONLY FROM THE REFERENCE CACHE
|
||||
*/
|
||||
final <T> T getCached(ByteArrayWrapper key) {
|
||||
final
|
||||
<T> T getCached(ByteArrayWrapper key) {
|
||||
// protected by lock
|
||||
|
||||
Metadata meta = this.memoryIndex.get(key);
|
||||
|
@ -236,7 +225,8 @@ public class StorageBase {
|
|||
/**
|
||||
* @return an object for a specified key form referenceCache FIRST, then from DISK
|
||||
*/
|
||||
final <T> T get(ByteArrayWrapper key) {
|
||||
final
|
||||
<T> T get(ByteArrayWrapper key) {
|
||||
// NOT protected by lock
|
||||
|
||||
Metadata meta = this.memoryIndex.get(key);
|
||||
|
@ -266,7 +256,7 @@ public class StorageBase {
|
|||
this.inflater.reset();
|
||||
this.file.seek(meta.dataPointer);
|
||||
|
||||
T readRecordData = meta.readData(this.kryo, this.inputStream);
|
||||
T readRecordData = Metadata.readData(this.serializationManager, this.inputStream);
|
||||
|
||||
if (readRecordData != null) {
|
||||
// now stuff it into our reference cache for future lookups!
|
||||
|
@ -281,10 +271,10 @@ public class StorageBase {
|
|||
|
||||
return readRecordData;
|
||||
} catch (KryoException e) {
|
||||
this.logger.error("Error while geting data from disk. Ignoring previous value.");
|
||||
this.logger.error("Error while getting data from disk. Ignoring previous value.");
|
||||
return null;
|
||||
} catch (Exception e) {
|
||||
this.logger.error("Error while geting data from disk", e);
|
||||
this.logger.error("Error while getting data from disk", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
@ -294,7 +284,8 @@ public class StorageBase {
|
|||
*
|
||||
* @return true if the delete was successful. False if there were problems deleting the data.
|
||||
*/
|
||||
final boolean delete(ByteArrayWrapper key) {
|
||||
final
|
||||
boolean delete(ByteArrayWrapper key) {
|
||||
// pending ops flushed (protected by lock)
|
||||
// not protected by lock
|
||||
Metadata delRec = this.memoryIndex.get(key);
|
||||
|
@ -312,11 +303,13 @@ public class StorageBase {
|
|||
/**
|
||||
* Closes the database and file.
|
||||
*/
|
||||
final void close() {
|
||||
final
|
||||
void close() {
|
||||
// pending ops flushed (protected by lock)
|
||||
// not protected by lock
|
||||
try {
|
||||
this.file.getFD().sync();
|
||||
this.file.getFD()
|
||||
.sync();
|
||||
this.file.close();
|
||||
this.memoryIndex.clear();
|
||||
|
||||
|
@ -344,22 +337,22 @@ public class StorageBase {
|
|||
/**
|
||||
* @return the file that backs this storage
|
||||
*/
|
||||
final File getFile() {
|
||||
final
|
||||
File getFile() {
|
||||
return this.baseFile;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Saves the given data to storage.
|
||||
* <p>
|
||||
* <p/>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
* <p>
|
||||
* <p/>
|
||||
* Will also save the object in a cache.
|
||||
*
|
||||
* @return the metadata for the saved object
|
||||
*/
|
||||
private final void save0(ByteArrayWrapper key, Object object, DeflaterOutputStream fileOutputStream, Deflater deflater) {
|
||||
private
|
||||
void save0(ByteArrayWrapper key, Object object, DeflaterOutputStream fileOutputStream, Deflater deflater) {
|
||||
deflater.reset();
|
||||
|
||||
Metadata metaData = this.memoryIndex.get(key);
|
||||
|
@ -370,19 +363,23 @@ public class StorageBase {
|
|||
try {
|
||||
if (currentRecordCount == 1) {
|
||||
// if we are the ONLY one, then we can do things differently.
|
||||
// just dump the data agian to disk.
|
||||
FileLock lock = this.file.getChannel().lock(this.dataPosition, Long.MAX_VALUE-this.dataPosition, false); // don't know how big it is, so max value it
|
||||
// just dump the data again to disk.
|
||||
FileLock lock = this.file.getChannel()
|
||||
.lock(this.dataPosition,
|
||||
Long.MAX_VALUE - this.dataPosition,
|
||||
false); // don't know how big it is, so max value it
|
||||
this.file.seek(this.dataPosition); // this is the end of the file, we know this ahead-of-time
|
||||
metaData.writeDataFast(this.kryo, object, fileOutputStream);
|
||||
Metadata.writeDataFast(this.serializationManager, object, file, fileOutputStream);
|
||||
// have to re-specify the capacity and size
|
||||
int sizeOfWrittenData = (int) (this.file.length() - this.dataPosition);
|
||||
|
||||
metaData.dataCapacity = sizeOfWrittenData;
|
||||
metaData.dataCount = sizeOfWrittenData;
|
||||
lock.release();
|
||||
} else {
|
||||
}
|
||||
else {
|
||||
// this is comparatively slow, since we serialize it first to get the size, then we put it in the file.
|
||||
ByteArrayOutputStream dataStream = getDataAsByteArray(this.kryo, object, deflater);
|
||||
ByteArrayOutputStream dataStream = getDataAsByteArray(this.serializationManager, object, deflater);
|
||||
|
||||
int size = dataStream.size();
|
||||
if (size > metaData.dataCapacity) {
|
||||
|
@ -404,10 +401,11 @@ public class StorageBase {
|
|||
} catch (IOException e) {
|
||||
this.logger.error("Error while saving data to disk", e);
|
||||
}
|
||||
} else {
|
||||
}
|
||||
else {
|
||||
try {
|
||||
// try to move the read head in order
|
||||
setRecordCount(this.file, currentRecordCount+1);
|
||||
setRecordCount(this.file, currentRecordCount + 1);
|
||||
|
||||
// This will make sure that there is room to write a new record. This is zero indexed.
|
||||
// this will skip around if moves occur
|
||||
|
@ -425,9 +423,10 @@ public class StorageBase {
|
|||
// save out the data. Because we KNOW that we are writing this to the end of the file,
|
||||
// there are some tricks we can use.
|
||||
|
||||
FileLock lock = this.file.getChannel().lock(length, Long.MAX_VALUE-length, false); // don't know how big it is, so max value it
|
||||
FileLock lock = this.file.getChannel()
|
||||
.lock(length, Long.MAX_VALUE - length, false); // don't know how big it is, so max value it
|
||||
this.file.seek(length); // this is the end of the file, we know this ahead-of-time
|
||||
metaData.writeDataFast(this.kryo, object, fileOutputStream);
|
||||
Metadata.writeDataFast(this.serializationManager, object, file, fileOutputStream);
|
||||
lock.release();
|
||||
|
||||
metaData.dataCount = deflater.getTotalOut();
|
||||
|
@ -445,7 +444,8 @@ public class StorageBase {
|
|||
}
|
||||
|
||||
|
||||
private ByteArrayOutputStream getDataAsByteArray(Kryo kryo, Object data, Deflater deflater) throws IOException {
|
||||
private static
|
||||
ByteArrayOutputStream getDataAsByteArray(SerializationManager kryo, Object data, Deflater deflater) throws IOException {
|
||||
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
OutputStream outputStream = new DeflaterOutputStream(byteArrayOutputStream, deflater);
|
||||
Output output = new Output(outputStream, 1024); // write 1024 at a time
|
||||
|
@ -484,23 +484,27 @@ public class StorageBase {
|
|||
|
||||
/**
|
||||
* "intelligent" move strategy.
|
||||
*
|
||||
* <p/>
|
||||
* we should increase by some weight (ie: .5) would increase the number of allocated
|
||||
* record headers by 50%, instead of just incrementing them by one at a time.
|
||||
*/
|
||||
private final int getWeightedNewRecordCount(int numberOfRecords) {
|
||||
int newNumberOfRecords = numberOfRecords + 1 + (int) (numberOfRecords * this.weight); // int used for rounding
|
||||
return newNumberOfRecords;
|
||||
private
|
||||
int getWeightedNewRecordCount(int numberOfRecords) {
|
||||
//noinspection AutoUnboxing
|
||||
return numberOfRecords + 1 + (int) (numberOfRecords * this.weight);
|
||||
}
|
||||
|
||||
|
||||
private void deleteRecordData(Metadata deletedRecord, int sizeOfDataToAdd) throws IOException {
|
||||
private
|
||||
void deleteRecordData(Metadata deletedRecord, int sizeOfDataToAdd) throws IOException {
|
||||
if (this.file.length() == deletedRecord.dataPointer + deletedRecord.dataCapacity) {
|
||||
// shrink file since this is the last record in the file
|
||||
FileLock lock = this.file.getChannel().lock(deletedRecord.dataPointer, Long.MAX_VALUE-deletedRecord.dataPointer, false);
|
||||
FileLock lock = this.file.getChannel()
|
||||
.lock(deletedRecord.dataPointer, Long.MAX_VALUE - deletedRecord.dataPointer, false);
|
||||
this.file.setLength(deletedRecord.dataPointer);
|
||||
lock.release();
|
||||
} else {
|
||||
}
|
||||
else {
|
||||
// we MIGHT be the FIRST record
|
||||
Metadata first = index_getMetaDataFromData(this.dataPosition);
|
||||
if (first == deletedRecord) {
|
||||
|
@ -515,22 +519,25 @@ public class StorageBase {
|
|||
long endIndexPointer = Metadata.getMetaDataPointer(newNumberOfRecords);
|
||||
|
||||
long endOfDataPointer = deletedRecord.dataPointer + deletedRecord.dataCapacity;
|
||||
long newEndOfDataPointer = endOfDataPointer-sizeOfDataToAdd;
|
||||
long newEndOfDataPointer = endOfDataPointer - sizeOfDataToAdd;
|
||||
|
||||
if (endIndexPointer < this.dataPosition && endIndexPointer <= newEndOfDataPointer) {
|
||||
// one option is to shrink the RECORD section to fit the new data
|
||||
setDataPosition(this.file, newEndOfDataPointer);
|
||||
} else {
|
||||
}
|
||||
else {
|
||||
// option two is to grow the RECORD section, and put the data at the end of the file
|
||||
setDataPosition(this.file, endOfDataPointer);
|
||||
}
|
||||
} else {
|
||||
}
|
||||
else {
|
||||
Metadata previous = index_getMetaDataFromData(deletedRecord.dataPointer - 1);
|
||||
if (previous != null) {
|
||||
// append space of deleted record onto previous record
|
||||
previous.dataCapacity += deletedRecord.dataCapacity;
|
||||
previous.writeDataInfo(this.file);
|
||||
} else {
|
||||
}
|
||||
else {
|
||||
// because there is no "previous", that means we MIGHT be the FIRST record
|
||||
// well, we're not the first record. which one is RIGHT before us?
|
||||
// it should be "previous", so something fucked up
|
||||
|
@ -540,11 +547,14 @@ public class StorageBase {
|
|||
}
|
||||
}
|
||||
|
||||
private void deleteRecordIndex(ByteArrayWrapper key, Metadata deleteRecord) throws IOException {
|
||||
private
|
||||
void deleteRecordIndex(ByteArrayWrapper key, Metadata deleteRecord) throws IOException {
|
||||
int currentNumRecords = this.memoryIndex.size();
|
||||
|
||||
if (deleteRecord.indexPosition != currentNumRecords - 1) {
|
||||
Metadata last = Metadata.readHeader(this.file, currentNumRecords - 1);
|
||||
assert last != null;
|
||||
|
||||
last.move(this.file, deleteRecord.indexPosition);
|
||||
}
|
||||
|
||||
|
@ -557,10 +567,12 @@ public class StorageBase {
|
|||
/**
|
||||
* Writes the number of records header to the file.
|
||||
*/
|
||||
private final void setVersionNumber(RandomAccessFile file, int versionNumber) throws IOException {
|
||||
private
|
||||
void setVersionNumber(RandomAccessFile file, int versionNumber) throws IOException {
|
||||
this.databaseVersion = versionNumber;
|
||||
|
||||
FileLock lock = this.file.getChannel().lock(VERSION_HEADER_LOCATION, 4, false);
|
||||
FileLock lock = this.file.getChannel()
|
||||
.lock(VERSION_HEADER_LOCATION, 4, false);
|
||||
file.seek(VERSION_HEADER_LOCATION);
|
||||
file.writeInt(versionNumber);
|
||||
lock.release();
|
||||
|
@ -569,10 +581,12 @@ public class StorageBase {
|
|||
/**
|
||||
* Writes the number of records header to the file.
|
||||
*/
|
||||
private final void setRecordCount(RandomAccessFile file, int numberOfRecords) throws IOException {
|
||||
private
|
||||
void setRecordCount(RandomAccessFile file, int numberOfRecords) throws IOException {
|
||||
this.numberOfRecords = numberOfRecords;
|
||||
|
||||
FileLock lock = this.file.getChannel().lock(NUM_RECORDS_HEADER_LOCATION, 4, false);
|
||||
FileLock lock = this.file.getChannel()
|
||||
.lock(NUM_RECORDS_HEADER_LOCATION, 4, false);
|
||||
file.seek(NUM_RECORDS_HEADER_LOCATION);
|
||||
file.writeInt(numberOfRecords);
|
||||
lock.release();
|
||||
|
@ -581,10 +595,12 @@ public class StorageBase {
|
|||
/**
|
||||
* Writes the data start position to the file.
|
||||
*/
|
||||
private final void setDataPosition(RandomAccessFile file, long dataPositionPointer) throws IOException {
|
||||
private
|
||||
void setDataPosition(RandomAccessFile file, long dataPositionPointer) throws IOException {
|
||||
this.dataPosition = dataPositionPointer;
|
||||
|
||||
FileLock lock = this.file.getChannel().lock(DATA_START_HEADER_LOCATION, 8, false);
|
||||
FileLock lock = this.file.getChannel()
|
||||
.lock(DATA_START_HEADER_LOCATION, 8, false);
|
||||
file.seek(DATA_START_HEADER_LOCATION);
|
||||
file.writeLong(dataPositionPointer);
|
||||
lock.release();
|
||||
|
@ -608,9 +624,12 @@ public class StorageBase {
|
|||
* of the record data of the RecordHeader which is returned. Returns null if the location is not part of a record.
|
||||
* (O(n) mem accesses)
|
||||
*/
|
||||
private final Metadata index_getMetaDataFromData(long targetFp) {
|
||||
Iterator<Metadata> iterator = this.memoryIndex.values().iterator();
|
||||
private
|
||||
Metadata index_getMetaDataFromData(long targetFp) {
|
||||
Iterator<Metadata> iterator = this.memoryIndex.values()
|
||||
.iterator();
|
||||
|
||||
//noinspection WhileLoopReplaceableByForEach
|
||||
while (iterator.hasNext()) {
|
||||
Metadata next = iterator.next();
|
||||
if (targetFp >= next.dataPointer && targetFp < next.dataPointer + next.dataCapacity) {
|
||||
|
@ -625,9 +644,11 @@ public class StorageBase {
|
|||
/**
|
||||
* Ensure index capacity. This operation makes sure the INDEX REGION is large enough to accommodate additional entries.
|
||||
*/
|
||||
private final void ensureIndexCapacity(RandomAccessFile file) throws IOException {
|
||||
private
|
||||
void ensureIndexCapacity(RandomAccessFile file) throws IOException {
|
||||
int numberOfRecords = this.numberOfRecords; // because we are zero indexed, this is ALSO the index where the record will START
|
||||
long endIndexPointer = Metadata.getMetaDataPointer(numberOfRecords + 1); // +1 because this is where that index will END (the start of the NEXT one)
|
||||
long endIndexPointer = Metadata.getMetaDataPointer(
|
||||
numberOfRecords + 1); // +1 because this is where that index will END (the start of the NEXT one)
|
||||
|
||||
// just set the data position to the end of the file, since we don't have any data yet.
|
||||
if (endIndexPointer > file.length() && numberOfRecords == 0) {
|
||||
|
|
|
@ -1,44 +1,118 @@
|
|||
package dorkbox.util;
|
||||
|
||||
import static org.junit.Assert.fail;
|
||||
import com.esotericsoftware.kryo.Kryo;
|
||||
import com.esotericsoftware.kryo.Registration;
|
||||
import com.esotericsoftware.kryo.Serializer;
|
||||
import com.esotericsoftware.kryo.io.Input;
|
||||
import com.esotericsoftware.kryo.io.Output;
|
||||
import dorkbox.util.storage.DiskStorageIfface;
|
||||
import dorkbox.util.storage.MakeStorage;
|
||||
import io.netty.buffer.ByteBuf;
|
||||
import org.junit.*;
|
||||
import org.junit.runners.MethodSorters;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.OptionalDataException;
|
||||
import java.util.Arrays;
|
||||
|
||||
import org.junit.After;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.FixMethodOrder;
|
||||
import org.junit.Test;
|
||||
import org.junit.runners.MethodSorters;
|
||||
|
||||
import dorkbox.util.storage.Storage;
|
||||
|
||||
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
|
||||
public class StorageTest {
|
||||
public
|
||||
class StorageTest {
|
||||
|
||||
private static final String TEST_DB = "sampleFile.records";
|
||||
private static final File TEST_DB = new File("sampleFile.records");
|
||||
private static final SerializationManager manager = new SerializationManager() {
|
||||
Kryo kryo = new Kryo();
|
||||
|
||||
static void log(String s) {
|
||||
@Override
|
||||
public
|
||||
boolean setReferences(final boolean references) {
|
||||
kryo.setReferences(references);
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
void setRegistrationRequired(final boolean registrationRequired) {
|
||||
kryo.setRegistrationRequired(registrationRequired);
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
void register(final Class<?> clazz) {
|
||||
kryo.register(clazz);
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
void register(final Class<?> clazz, final Serializer<?> serializer) {
|
||||
kryo.register(clazz, serializer);
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
Registration register(final Class<?> type, final Serializer<?> serializer, final int id) {
|
||||
kryo.register(type, serializer, id);
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
void write(final ByteBuf buffer, final Object message) {
|
||||
final Output output = new Output();
|
||||
writeClassAndObject(output, message);
|
||||
buffer.writeBytes(output.getBuffer());
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
Object read(final ByteBuf buffer, final int length) {
|
||||
final Input input = new Input();
|
||||
buffer.readBytes(input.getBuffer());
|
||||
|
||||
final Object o = readClassAndObject(input);
|
||||
buffer.skipBytes(input.position());
|
||||
|
||||
return o;
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
void writeClassAndObject(final Output output, final Object value) {
|
||||
kryo.writeClassAndObject(output, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
public
|
||||
Object readClassAndObject(final Input input) {
|
||||
return kryo.readClassAndObject(input);
|
||||
}
|
||||
};
|
||||
|
||||
static
|
||||
void log(String s) {
|
||||
System.err.println(s);
|
||||
}
|
||||
|
||||
@Before
|
||||
public void deleteDB() {
|
||||
Storage.delete(new File(TEST_DB));
|
||||
public
|
||||
void deleteDB() {
|
||||
MakeStorage.delete(TEST_DB);
|
||||
}
|
||||
|
||||
@After
|
||||
public void delete2DB() {
|
||||
Storage.delete(new File(TEST_DB));
|
||||
public
|
||||
void delete2DB() {
|
||||
MakeStorage.delete(TEST_DB);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testCreateDB() throws IOException {
|
||||
Storage storage = Storage.open(TEST_DB);
|
||||
public
|
||||
void testCreateDB() throws IOException {
|
||||
DiskStorageIfface storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
|
||||
int numberOfRecords1 = storage.size();
|
||||
long size1 = storage.getFileSize();
|
||||
|
@ -46,67 +120,84 @@ public class StorageTest {
|
|||
Assert.assertEquals("count is not correct", numberOfRecords1, 0);
|
||||
Assert.assertEquals("size is not correct", size1, 208L); // NOTE this will change based on the data size added!
|
||||
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
|
||||
storage = Storage.open(TEST_DB);
|
||||
storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
int numberOfRecords2 = storage.size();
|
||||
long size2 = storage.getFileSize();
|
||||
|
||||
Assert.assertEquals("Record count is not the same", numberOfRecords1, numberOfRecords2);
|
||||
Assert.assertEquals("size is not the same", size1, size2);
|
||||
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testAddAsOne() throws IOException, ClassNotFoundException {
|
||||
public
|
||||
void testAddAsOne() throws IOException, ClassNotFoundException {
|
||||
int total = 100;
|
||||
|
||||
try {
|
||||
Storage storage = Storage.open(TEST_DB);
|
||||
for (int i=0;i<total;i++) {
|
||||
DiskStorageIfface storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
for (int i = 0; i < total; i++) {
|
||||
add(storage, i);
|
||||
}
|
||||
|
||||
Storage.close(storage);
|
||||
storage = Storage.open(TEST_DB);
|
||||
for (int i=0;i<total;i++) {
|
||||
MakeStorage.close(storage);
|
||||
storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
for (int i = 0; i < total; i++) {
|
||||
String record1Data = createData(i);
|
||||
String readRecord = readRecord(storage, i);
|
||||
|
||||
Assert.assertEquals("Object is not the same", record1Data, readRecord);
|
||||
}
|
||||
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
fail("Error!");
|
||||
Assert.fail("Error!");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddNoKeyRecords() throws IOException, ClassNotFoundException {
|
||||
public
|
||||
void testAddNoKeyRecords() throws IOException, ClassNotFoundException {
|
||||
int total = 100;
|
||||
|
||||
try {
|
||||
Storage storage = Storage.open(TEST_DB);
|
||||
for (int i=0;i<total;i++) {
|
||||
DiskStorageIfface storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
for (int i = 0; i < total; i++) {
|
||||
log("adding record " + i + "...");
|
||||
String addRecord = createData(i);
|
||||
storage.save(addRecord);
|
||||
storage.put(addRecord);
|
||||
|
||||
log("reading record " + i + "...");
|
||||
String readData = storage.get();
|
||||
|
||||
Assert.assertEquals("Object is not the same", addRecord, readData);
|
||||
}
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
|
||||
storage = Storage.open(TEST_DB);
|
||||
storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
|
||||
String dataCheck = createData(total-1);
|
||||
log("reading record " + (total-1) + "...");
|
||||
String dataCheck = createData(total - 1);
|
||||
log("reading record " + (total - 1) + "...");
|
||||
String readData = storage.get();
|
||||
|
||||
Assert.assertEquals("Object is not the same", dataCheck, readData);
|
||||
|
@ -117,101 +208,123 @@ public class StorageTest {
|
|||
Assert.assertEquals("count is not correct", numberOfRecords1, 1);
|
||||
Assert.assertEquals("size is not correct", size1, 235L); // NOTE this will change based on the data size added!
|
||||
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
fail("Error!");
|
||||
Assert.fail("Error!");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddRecords_DelaySaveA() throws IOException, ClassNotFoundException {
|
||||
public
|
||||
void testAddRecords_DelaySaveA() throws IOException, ClassNotFoundException {
|
||||
int total = 100;
|
||||
|
||||
try {
|
||||
Storage storage = Storage.open(TEST_DB);
|
||||
for (int i=0;i<total;i++) {
|
||||
DiskStorageIfface storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
for (int i = 0; i < total; i++) {
|
||||
add(storage, i);
|
||||
}
|
||||
|
||||
synchronized (Thread.currentThread()) {
|
||||
Thread.currentThread().wait(storage.getSaveDelay() + 1000L);
|
||||
Thread.currentThread()
|
||||
.wait(storage.getSaveDelay() + 1000L);
|
||||
}
|
||||
|
||||
for (int i=0;i<total;i++) {
|
||||
for (int i = 0; i < total; i++) {
|
||||
String record1Data = createData(i);
|
||||
String readRecord = readRecord(storage, i);
|
||||
|
||||
Assert.assertEquals("Object is not the same", record1Data, readRecord);
|
||||
}
|
||||
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
|
||||
storage = Storage.open(TEST_DB);
|
||||
for (int i=0;i<total;i++) {
|
||||
storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
for (int i = 0; i < total; i++) {
|
||||
String dataCheck = createData(i);
|
||||
String readRecord = readRecord(storage, i);
|
||||
|
||||
Assert.assertEquals("Object is not the same", dataCheck, readRecord);
|
||||
}
|
||||
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
fail("Error!");
|
||||
Assert.fail("Error!");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddRecords_DelaySaveB() throws IOException, ClassNotFoundException {
|
||||
public
|
||||
void testAddRecords_DelaySaveB() throws IOException, ClassNotFoundException {
|
||||
int total = 100;
|
||||
|
||||
try {
|
||||
Storage storage = Storage.open(TEST_DB);
|
||||
for (int i=0;i<total;i++) {
|
||||
DiskStorageIfface storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
for (int i = 0; i < total; i++) {
|
||||
add(storage, i);
|
||||
}
|
||||
|
||||
for (int i=0;i<total;i++) {
|
||||
for (int i = 0; i < total; i++) {
|
||||
String record1Data = createData(i);
|
||||
String readRecord = readRecord(storage, i);
|
||||
|
||||
Assert.assertEquals("Object is not the same", record1Data, readRecord);
|
||||
}
|
||||
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
|
||||
storage = Storage.open(TEST_DB);
|
||||
for (int i=0;i<total;i++) {
|
||||
storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
for (int i = 0; i < total; i++) {
|
||||
String dataCheck = createData(i);
|
||||
String readRecord = readRecord(storage, i);
|
||||
|
||||
Assert.assertEquals("Object is not the same", dataCheck, readRecord);
|
||||
}
|
||||
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
fail("Error!");
|
||||
Assert.fail("Error!");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testLoadRecords() throws IOException, ClassNotFoundException {
|
||||
public
|
||||
void testLoadRecords() throws IOException, ClassNotFoundException {
|
||||
int total = 100;
|
||||
|
||||
try {
|
||||
Storage storage = Storage.open(TEST_DB);
|
||||
for (int i=0;i<total;i++) {
|
||||
DiskStorageIfface storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
for (int i = 0; i < total; i++) {
|
||||
String addRecord = add(storage, i);
|
||||
String readRecord = readRecord(storage, i);
|
||||
|
||||
Assert.assertEquals("Object is not the same", addRecord, readRecord);
|
||||
}
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
|
||||
storage = Storage.open(TEST_DB);
|
||||
for (int i=0;i<total;i++) {
|
||||
storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
for (int i = 0; i < total; i++) {
|
||||
String dataCheck = createData(i);
|
||||
String readRecord = readRecord(storage, i);
|
||||
|
||||
|
@ -223,43 +336,53 @@ public class StorageTest {
|
|||
String createKey = createKey(63);
|
||||
makeData(data);
|
||||
|
||||
storage.save(createKey, data);
|
||||
storage.put(createKey, data);
|
||||
|
||||
Data data2 = new Data();
|
||||
storage.load(createKey, data2);
|
||||
Assert.assertEquals("Object is not the same", data, data2);
|
||||
|
||||
Storage.close(storage);
|
||||
storage = Storage.open(TEST_DB);
|
||||
MakeStorage.close(storage);
|
||||
storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
|
||||
data2 = new Data();
|
||||
storage.load(createKey, data2);
|
||||
Assert.assertEquals("Object is not the same", data, data2);
|
||||
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
fail("Error!");
|
||||
Assert.fail("Error!");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testAddRecordsDelete1Record() throws IOException, ClassNotFoundException {
|
||||
public
|
||||
void testAddRecordsDelete1Record() throws IOException, ClassNotFoundException {
|
||||
int total = 100;
|
||||
|
||||
try {
|
||||
Storage storage = Storage.open(TEST_DB);
|
||||
for (int i=0;i<total;i++) {
|
||||
DiskStorageIfface storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
for (int i = 0; i < total; i++) {
|
||||
String addRecord = add(storage, i);
|
||||
String readRecord = readRecord(storage, i);
|
||||
|
||||
Assert.assertEquals("Object is not the same", addRecord, readRecord);
|
||||
}
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
|
||||
storage = Storage.open(TEST_DB);
|
||||
for (int i=0;i<total;i++) {
|
||||
storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
for (int i = 0; i < total; i++) {
|
||||
String dataCheck = createData(i);
|
||||
String readRecord = readRecord(storage, i);
|
||||
|
||||
|
@ -276,7 +399,7 @@ public class StorageTest {
|
|||
|
||||
|
||||
if (storage.contains(createKey(3))) {
|
||||
fail("record NOT successfully deleted.");
|
||||
Assert.fail("record NOT successfully deleted.");
|
||||
}
|
||||
|
||||
// now we add 3 back
|
||||
|
@ -285,9 +408,12 @@ public class StorageTest {
|
|||
|
||||
Assert.assertEquals("Object is not the same", dataCheck, addRecord);
|
||||
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
|
||||
storage = Storage.open(TEST_DB);
|
||||
storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
|
||||
// check 9 again
|
||||
readRecord = readRecord(storage, 9);
|
||||
|
@ -301,110 +427,134 @@ public class StorageTest {
|
|||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
fail("Error!");
|
||||
Assert.fail("Error!");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testUpdateRecords() throws IOException, ClassNotFoundException {
|
||||
public
|
||||
void testUpdateRecords() throws IOException, ClassNotFoundException {
|
||||
int total = 100;
|
||||
|
||||
try {
|
||||
Storage storage = Storage.open(TEST_DB);
|
||||
for (int i=0;i<total;i++) {
|
||||
DiskStorageIfface storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
for (int i = 0; i < total; i++) {
|
||||
String addRecord = add(storage, i);
|
||||
String readRecord = readRecord(storage, i);
|
||||
|
||||
Assert.assertEquals("Object is not the same", addRecord, readRecord);
|
||||
}
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
|
||||
storage = Storage.open(TEST_DB);
|
||||
storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
|
||||
String updateRecord = updateRecord(storage, 3, createData(3) + "new");
|
||||
String readRecord = readRecord(storage, 3);
|
||||
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
|
||||
|
||||
Storage.close(storage);
|
||||
storage = Storage.open(TEST_DB);
|
||||
MakeStorage.close(storage);
|
||||
storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
|
||||
readRecord = readRecord(storage, 3);
|
||||
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
|
||||
|
||||
updateRecord = updateRecord(storage, 3, createData(3));
|
||||
|
||||
Storage.close(storage);
|
||||
storage = Storage.open(TEST_DB);
|
||||
MakeStorage.close(storage);
|
||||
storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
|
||||
readRecord = readRecord(storage, 3);
|
||||
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
|
||||
|
||||
Storage.close(storage);
|
||||
storage = Storage.open(TEST_DB);
|
||||
MakeStorage.close(storage);
|
||||
storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
|
||||
updateRecord = updateRecord(storage, 0, createData(0) + "new");
|
||||
readRecord = readRecord(storage, 0);
|
||||
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
|
||||
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
fail("Error!");
|
||||
Assert.fail("Error!");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testSaveAllRecords() throws IOException, ClassNotFoundException {
|
||||
public
|
||||
void testSaveAllRecords() throws IOException, ClassNotFoundException {
|
||||
int total = 100;
|
||||
|
||||
try {
|
||||
Storage storage = Storage.open(TEST_DB);
|
||||
for (int i=0;i<total;i++) {
|
||||
DiskStorageIfface storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
for (int i = 0; i < total; i++) {
|
||||
Data data = new Data();
|
||||
makeData(data);
|
||||
String createKey = createKey(i);
|
||||
|
||||
storage.save(createKey, data);
|
||||
storage.put(createKey, data);
|
||||
}
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
|
||||
Data data = new Data();
|
||||
makeData(data);
|
||||
|
||||
storage = Storage.open(TEST_DB);
|
||||
for (int i=0;i<total;i++) {
|
||||
storage = MakeStorage.Disk()
|
||||
.file(TEST_DB)
|
||||
.serializer(manager)
|
||||
.make();
|
||||
for (int i = 0; i < total; i++) {
|
||||
String createKey = createKey(i);
|
||||
|
||||
Data data2 = new Data();
|
||||
storage.load(createKey, data2);
|
||||
Assert.assertEquals("Object is not the same", data, data2);
|
||||
}
|
||||
Storage.close(storage);
|
||||
MakeStorage.close(storage);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
fail("Error!");
|
||||
Assert.fail("Error!");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
private String createData(int number) {
|
||||
private static
|
||||
String createData(int number) {
|
||||
return number + " data for record # " + number;
|
||||
}
|
||||
|
||||
public String add(Storage storage, int number) throws IOException {
|
||||
public static
|
||||
String add(DiskStorageIfface storage, int number) throws IOException {
|
||||
String record1Data = createData(number);
|
||||
String record1Key = createKey(number);
|
||||
|
||||
log("adding record " + number + "...");
|
||||
storage.save(record1Key, record1Data);
|
||||
storage.put(record1Key, record1Data);
|
||||
return record1Data;
|
||||
}
|
||||
|
||||
public String readRecord(Storage storage, int number) throws OptionalDataException, ClassNotFoundException, IOException {
|
||||
public static
|
||||
String readRecord(DiskStorageIfface storage, int number) throws ClassNotFoundException, IOException {
|
||||
String record1Key = createKey(number);
|
||||
|
||||
log("reading record " + number + "...");
|
||||
|
@ -414,30 +564,34 @@ public class StorageTest {
|
|||
return readData;
|
||||
}
|
||||
|
||||
public void deleteRecord(Storage storage, int nNumber) throws OptionalDataException, ClassNotFoundException, IOException {
|
||||
public static
|
||||
void deleteRecord(DiskStorageIfface storage, int nNumber) throws ClassNotFoundException, IOException {
|
||||
String record1Key = createKey(nNumber);
|
||||
|
||||
log("deleting record " + nNumber + "...");
|
||||
storage.delete(record1Key);
|
||||
}
|
||||
|
||||
private String updateRecord(Storage storage, int number, String newData) throws IOException {
|
||||
private static
|
||||
String updateRecord(DiskStorageIfface storage, int number, String newData) throws IOException {
|
||||
String record1Key = createKey(number);
|
||||
|
||||
log("updating record " + number + "...");
|
||||
storage.save(record1Key, newData);
|
||||
storage.put(record1Key, newData);
|
||||
|
||||
return newData;
|
||||
}
|
||||
|
||||
private String createKey(int number) {
|
||||
private static
|
||||
String createKey(int number) {
|
||||
return "foo" + number;
|
||||
}
|
||||
|
||||
|
||||
// from kryo unit test.
|
||||
private void makeData(Data data) {
|
||||
StringBuilder buffer = new StringBuilder();
|
||||
private static
|
||||
void makeData(Data data) {
|
||||
StringBuilder buffer = new StringBuilder(128);
|
||||
for (int i = 0; i < 3; i++) {
|
||||
buffer.append('a');
|
||||
}
|
||||
|
@ -446,8 +600,7 @@ public class StorageTest {
|
|||
data.strings = new String[] {"ab012", "", null, "!@#$", "<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>"};
|
||||
data.ints = new int[] {-1234567, 1234567, -1, 0, 1, Integer.MAX_VALUE, Integer.MIN_VALUE};
|
||||
data.shorts = new short[] {-12345, 12345, -1, 0, 1, Short.MAX_VALUE, Short.MIN_VALUE};
|
||||
data.floats = new float[] {0, -0, 1, -1, 123456, -123456, 0.1f, 0.2f, -0.3f, (float)Math.PI, Float.MAX_VALUE,
|
||||
Float.MIN_VALUE};
|
||||
data.floats = new float[] {0, -0, 1, -1, 123456, -123456, 0.1f, 0.2f, -0.3f, (float) Math.PI, Float.MAX_VALUE, Float.MIN_VALUE};
|
||||
data.doubles = new double[] {0, -0, 1, -1, 123456, -123456, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE, Double.MIN_VALUE};
|
||||
data.longs = new long[] {0, -0, 1, -1, 123456, -123456, 99999999999l, -99999999999l, Long.MAX_VALUE, Long.MIN_VALUE};
|
||||
data.bytes = new byte[] {-123, 123, -1, 0, 1, Byte.MAX_VALUE, Byte.MIN_VALUE};
|
||||
|
@ -455,17 +608,17 @@ public class StorageTest {
|
|||
data.booleans = new boolean[] {true, false};
|
||||
data.Ints = new Integer[] {-1234567, 1234567, -1, 0, 1, Integer.MAX_VALUE, Integer.MIN_VALUE};
|
||||
data.Shorts = new Short[] {-12345, 12345, -1, 0, 1, Short.MAX_VALUE, Short.MIN_VALUE};
|
||||
data.Floats = new Float[] {0f, -0f, 1f, -1f, 123456f, -123456f, 0.1f, 0.2f, -0.3f, (float)Math.PI, Float.MAX_VALUE,
|
||||
data.Floats = new Float[] {0f, -0f, 1f, -1f, 123456f, -123456f, 0.1f, 0.2f, -0.3f, (float) Math.PI, Float.MAX_VALUE,
|
||||
Float.MIN_VALUE};
|
||||
data.Doubles = new Double[] {0d, -0d, 1d, -1d, 123456d, -123456d, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE,
|
||||
Double.MIN_VALUE};
|
||||
data.Doubles = new Double[] {0d, -0d, 1d, -1d, 123456d, -123456d, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE, Double.MIN_VALUE};
|
||||
data.Longs = new Long[] {0l, -0l, 1l, -1l, 123456l, -123456l, 99999999999l, -99999999999l, Long.MAX_VALUE, Long.MIN_VALUE};
|
||||
data.Bytes = new Byte[] {-123, 123, -1, 0, 1, Byte.MAX_VALUE, Byte.MIN_VALUE};
|
||||
data.Chars = new Character[] {32345, 12345, 0, 1, 63, Character.MAX_VALUE, Character.MIN_VALUE};
|
||||
data.Booleans = new Boolean[] {true, false};
|
||||
}
|
||||
|
||||
public static class Data {
|
||||
public static
|
||||
class Data {
|
||||
public String string;
|
||||
public String[] strings;
|
||||
public int[] ints;
|
||||
|
@ -485,11 +638,13 @@ public class StorageTest {
|
|||
public Character[] Chars;
|
||||
public Boolean[] Booleans;
|
||||
|
||||
public Data() {
|
||||
public
|
||||
Data() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode () {
|
||||
public
|
||||
int hashCode() {
|
||||
final int prime = 31;
|
||||
int result = 1;
|
||||
result = prime * result + Arrays.hashCode(this.Booleans);
|
||||
|
@ -514,7 +669,8 @@ public class StorageTest {
|
|||
}
|
||||
|
||||
@Override
|
||||
public boolean equals (Object obj) {
|
||||
public
|
||||
boolean equals(Object obj) {
|
||||
if (this == obj) {
|
||||
return true;
|
||||
}
|
||||
|
@ -524,7 +680,7 @@ public class StorageTest {
|
|||
if (getClass() != obj.getClass()) {
|
||||
return false;
|
||||
}
|
||||
Data other = (Data)obj;
|
||||
Data other = (Data) obj;
|
||||
if (!Arrays.equals(this.Booleans, other.Booleans)) {
|
||||
return false;
|
||||
}
|
||||
|
@ -577,7 +733,8 @@ public class StorageTest {
|
|||
if (other.string != null) {
|
||||
return false;
|
||||
}
|
||||
} else if (!this.string.equals(other.string)) {
|
||||
}
|
||||
else if (!this.string.equals(other.string)) {
|
||||
return false;
|
||||
}
|
||||
if (!Arrays.equals(this.strings, other.strings)) {
|
||||
|
@ -587,7 +744,8 @@ public class StorageTest {
|
|||
}
|
||||
|
||||
@Override
|
||||
public String toString () {
|
||||
public
|
||||
String toString() {
|
||||
return "Data";
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue
Block a user