wip - creating disk or memory based storage
This commit is contained in:
parent
927a5aece9
commit
08f7872a40
516
Dorkbox-Util/src/dorkbox/util/storage/DiskStorage.java
Normal file
516
Dorkbox-Util/src/dorkbox/util/storage/DiskStorage.java
Normal file
|
@ -0,0 +1,516 @@
|
||||||
|
/*
|
||||||
|
* Copyright 2014 dorkbox, llc
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
package dorkbox.util.storage;
|
||||||
|
|
||||||
|
import dorkbox.util.DelayTimer;
|
||||||
|
import dorkbox.util.SerializationManager;
|
||||||
|
import dorkbox.util.bytes.ByteArrayWrapper;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
import java.util.concurrent.locks.ReentrantLock;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Nothing spectacular about this storage -- it allows for persistent storage of objects to disk.
|
||||||
|
* <p/>
|
||||||
|
* Be wary of opening the database file in different instances. Even with file-locks, you can corrupt the data.
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unused")
|
||||||
|
public
|
||||||
|
class DiskStorage implements DiskStorageIfface {
|
||||||
|
private final DelayTimer timer;
|
||||||
|
private final ByteArrayWrapper defaultKey;
|
||||||
|
private final StorageBase storage;
|
||||||
|
|
||||||
|
private final AtomicInteger references = new AtomicInteger(1);
|
||||||
|
private final ReentrantLock actionLock = new ReentrantLock();
|
||||||
|
private final AtomicBoolean isOpen = new AtomicBoolean(false);
|
||||||
|
private volatile long milliSeconds = 3000L;
|
||||||
|
|
||||||
|
private volatile Map<ByteArrayWrapper, Object> actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates or opens a new database file.
|
||||||
|
*/
|
||||||
|
protected
|
||||||
|
DiskStorage(File storageFile, SerializationManager serializationManager) throws IOException {
|
||||||
|
this.storage = new StorageBase(storageFile, serializationManager);
|
||||||
|
this.defaultKey = ByteArrayWrapper.wrap("");
|
||||||
|
|
||||||
|
this.timer = new DelayTimer("Storage Writer", false, new DelayTimer.Callback() {
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void execute() {
|
||||||
|
Map<ByteArrayWrapper, Object> actions = DiskStorage.this.actionMap;
|
||||||
|
|
||||||
|
ReentrantLock actionLock2 = DiskStorage.this.actionLock;
|
||||||
|
|
||||||
|
try {
|
||||||
|
actionLock2.lock();
|
||||||
|
|
||||||
|
// do a fast swap on the actionMap.
|
||||||
|
DiskStorage.this.actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
|
||||||
|
} finally {
|
||||||
|
actionLock2.unlock();
|
||||||
|
}
|
||||||
|
|
||||||
|
DiskStorage.this.storage.doActionThings(actions);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
this.isOpen.set(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the number of objects in the database.
|
||||||
|
* <p/>
|
||||||
|
* SLOW because this must save all data to disk first!
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
int size() {
|
||||||
|
if (!this.isOpen.get()) {
|
||||||
|
throw new RuntimeException("Unable to act on closed storage");
|
||||||
|
}
|
||||||
|
|
||||||
|
// flush actions
|
||||||
|
// timer action runs on THIS thread, not timer thread
|
||||||
|
this.timer.delay(0L);
|
||||||
|
|
||||||
|
return this.storage.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks if there is a object corresponding to the given key.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
boolean contains(String key) {
|
||||||
|
if (!this.isOpen.get()) {
|
||||||
|
throw new RuntimeException("Unable to act on closed storage");
|
||||||
|
}
|
||||||
|
|
||||||
|
final ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
|
||||||
|
// check if our pending actions has it, or if our storage index has it
|
||||||
|
return this.actionMap.containsKey(wrap) || this.storage.contains(wrap);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a object using the default (blank) key, and casts it to the expected class
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
<T> T get() {
|
||||||
|
return get0(this.defaultKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a object using the specific key, and casts it to the expected class
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
<T> T get(String key) {
|
||||||
|
return get0(ByteArrayWrapper.wrap(key));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a object using the specific key, and casts it to the expected class
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
<T> T get(byte[] key) {
|
||||||
|
return get0(ByteArrayWrapper.wrap(key));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a object using the specific key, and casts it to the expected class
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
<T> T get(ByteArrayWrapper key) {
|
||||||
|
return get0(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Uses the DEFAULT key ("") to return saved data.
|
||||||
|
* <p/>
|
||||||
|
* This will check to see if there is an associated key for that data, if not - it will use data as the default
|
||||||
|
*
|
||||||
|
* @param data The data that will hold the copy of the data from disk
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
<T> T load(T data) throws IOException {
|
||||||
|
return load(this.defaultKey, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the saved data for the specified key.
|
||||||
|
*
|
||||||
|
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
<T> T load(String key, T data) throws IOException {
|
||||||
|
ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
|
||||||
|
|
||||||
|
return load(wrap, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the saved data for the specified key.
|
||||||
|
*
|
||||||
|
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
<T> T load(byte[] key, T data) throws IOException {
|
||||||
|
return load(ByteArrayWrapper.wrap(key), data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the saved data for the specified key.
|
||||||
|
*
|
||||||
|
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
public
|
||||||
|
<T> T load(ByteArrayWrapper key, T data) throws IOException {
|
||||||
|
Object source = get0(key);
|
||||||
|
|
||||||
|
if (source == null) {
|
||||||
|
// returned was null, so we should take value as the default
|
||||||
|
put(key, data);
|
||||||
|
return data;
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
final Class<?> expectedClass = data.getClass();
|
||||||
|
final Class<?> savedCLass = source.getClass();
|
||||||
|
|
||||||
|
if (!expectedClass.isAssignableFrom(savedCLass)) {
|
||||||
|
throw new IOException("Saved value type '" + source.getClass() + "' is different that expected value");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return (T) source;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a object from pending or from storage
|
||||||
|
*/
|
||||||
|
private
|
||||||
|
<T> T get0(ByteArrayWrapper key) {
|
||||||
|
if (!this.isOpen.get()) {
|
||||||
|
throw new RuntimeException("Unable to act on closed storage");
|
||||||
|
}
|
||||||
|
|
||||||
|
// if the object in is pending, we get it from there
|
||||||
|
try {
|
||||||
|
this.actionLock.lock();
|
||||||
|
|
||||||
|
Object object = this.actionMap.get(key);
|
||||||
|
|
||||||
|
if (object != null) {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
T returnObject = (T) object;
|
||||||
|
return returnObject;
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
this.actionLock.unlock();
|
||||||
|
}
|
||||||
|
|
||||||
|
// not found, so we have to go find it on disk
|
||||||
|
return this.storage.get(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Saves the given data to storage with the associated key.
|
||||||
|
* <p/>
|
||||||
|
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||||
|
* deleting the old data and adding the new.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
void put(String key, Object object) {
|
||||||
|
put(ByteArrayWrapper.wrap(key), object);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Saves the given data to storage with the associated key.
|
||||||
|
* <p/>
|
||||||
|
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||||
|
* deleting the old data and adding the new.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
void put(byte[] key, Object object) {
|
||||||
|
put(ByteArrayWrapper.wrap(key), object);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Saves the given data to storage with the associated key.
|
||||||
|
* <p/>
|
||||||
|
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||||
|
* deleting the old data and adding the new.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
void put(ByteArrayWrapper key, Object object) {
|
||||||
|
if (!this.isOpen.get()) {
|
||||||
|
throw new RuntimeException("Unable to act on closed storage");
|
||||||
|
}
|
||||||
|
|
||||||
|
action(key, object);
|
||||||
|
|
||||||
|
// timer action runs on TIMER thread, not this thread
|
||||||
|
this.timer.delay(this.milliSeconds);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Adds the given object to the storage using a default (blank) key, OR -- if it has been registered, using it's registered key
|
||||||
|
* <p/>
|
||||||
|
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||||
|
* deleting the old data and adding the new.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
void put(Object object) {
|
||||||
|
if (!this.isOpen.get()) {
|
||||||
|
throw new RuntimeException("Unable to act on closed storage");
|
||||||
|
}
|
||||||
|
|
||||||
|
action(this.defaultKey, object);
|
||||||
|
|
||||||
|
// timer action runs on TIMER thread, not this thread
|
||||||
|
this.timer.delay(this.milliSeconds);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deletes an object from storage. To ALSO remove from the cache, use unRegister(key)
|
||||||
|
*
|
||||||
|
* @return true if the delete was successful. False if there were problems deleting the data.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
boolean delete(String key) {
|
||||||
|
if (!this.isOpen.get()) {
|
||||||
|
throw new RuntimeException("Unable to act on closed storage");
|
||||||
|
}
|
||||||
|
|
||||||
|
ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
|
||||||
|
|
||||||
|
// timer action runs on THIS thread, not timer thread
|
||||||
|
this.timer.delay(0L);
|
||||||
|
|
||||||
|
return this.storage.delete(wrap);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Closes the database and file.
|
||||||
|
*/
|
||||||
|
void close() {
|
||||||
|
// timer action runs on THIS thread, not timer thread
|
||||||
|
this.timer.delay(0L);
|
||||||
|
|
||||||
|
// have to "close" it after we run the timer!
|
||||||
|
this.isOpen.set(false);
|
||||||
|
|
||||||
|
this.storage.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return the file that backs this storage
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
File getFile() {
|
||||||
|
return this.storage.getFile();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the backing file size.
|
||||||
|
*
|
||||||
|
* @return -1 if there was an error
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
long getFileSize() {
|
||||||
|
// timer action runs on THIS thread, not timer thread
|
||||||
|
this.timer.delay(0L);
|
||||||
|
|
||||||
|
return this.storage.getFileSize();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return true if there are objects queued to be written?
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
boolean hasWriteWaiting() {
|
||||||
|
if (!this.isOpen.get()) {
|
||||||
|
throw new RuntimeException("Unable to act on closed storage");
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.timer.isWaiting();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return the delay in milliseconds this will wait after the last action to flush the data to the disk
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
long getSaveDelay() {
|
||||||
|
return this.milliSeconds;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param milliSeconds milliseconds to wait
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
void setSaveDelay(long milliSeconds) {
|
||||||
|
if (!this.isOpen.get()) {
|
||||||
|
throw new RuntimeException("Unable to act on closed storage");
|
||||||
|
}
|
||||||
|
|
||||||
|
this.milliSeconds = milliSeconds;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return the version of data stored in the database
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
int getVersion() {
|
||||||
|
return this.storage.getVersion();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets the version of data stored in the database
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
void setVersion(int version) {
|
||||||
|
this.storage.setVersion(version);
|
||||||
|
}
|
||||||
|
|
||||||
|
private
|
||||||
|
void action(ByteArrayWrapper key, Object object) {
|
||||||
|
try {
|
||||||
|
this.actionLock.lock();
|
||||||
|
|
||||||
|
// push action to map
|
||||||
|
this.actionMap.put(key, object);
|
||||||
|
} finally {
|
||||||
|
this.actionLock.unlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void increaseReference() {
|
||||||
|
this.references.incrementAndGet();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* return true when this is the last reference
|
||||||
|
*/
|
||||||
|
boolean decrementReference() {
|
||||||
|
return this.references.decrementAndGet() <= 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected
|
||||||
|
Object clone() throws CloneNotSupportedException {
|
||||||
|
return super.clone();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Save the storage to disk, immediately.
|
||||||
|
* <p/>
|
||||||
|
* This will save the ALL of the pending save actions to the file
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public final
|
||||||
|
void commit() {
|
||||||
|
if (!this.isOpen.get()) {
|
||||||
|
throw new RuntimeException("Unable to act on closed storage");
|
||||||
|
}
|
||||||
|
|
||||||
|
// timer action runs on THIS thread, not timer thread
|
||||||
|
this.timer.delay(0L);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Save the storage to disk, immediately.
|
||||||
|
* <p/>
|
||||||
|
* This will save the ALL of the pending save actions to the file
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void commit(final String key, final Object object) {
|
||||||
|
if (!this.isOpen.get()) {
|
||||||
|
throw new RuntimeException("Unable to act on closed storage");
|
||||||
|
}
|
||||||
|
|
||||||
|
action(ByteArrayWrapper.wrap(key), object);
|
||||||
|
|
||||||
|
// timer action runs on THIS thread, not timer thread
|
||||||
|
this.timer.delay(0L);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Save the storage to disk, immediately.
|
||||||
|
* <p/>
|
||||||
|
* This will save the ALL of the pending save actions to the file
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void commit(final byte[] key, final Object object) {
|
||||||
|
if (!this.isOpen.get()) {
|
||||||
|
throw new RuntimeException("Unable to act on closed storage");
|
||||||
|
}
|
||||||
|
|
||||||
|
action(ByteArrayWrapper.wrap(key), object);
|
||||||
|
|
||||||
|
// timer action runs on THIS thread, not timer thread
|
||||||
|
this.timer.delay(0L);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Save the storage to disk, immediately.
|
||||||
|
* <p/>
|
||||||
|
* This will save the ALL of the pending save actions to the file
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void commit(final ByteArrayWrapper key, final Object object) {
|
||||||
|
if (!this.isOpen.get()) {
|
||||||
|
throw new RuntimeException("Unable to act on closed storage");
|
||||||
|
}
|
||||||
|
|
||||||
|
action(key, object);
|
||||||
|
|
||||||
|
// timer action runs on THIS thread, not timer thread
|
||||||
|
this.timer.delay(0L);
|
||||||
|
}
|
||||||
|
}
|
177
Dorkbox-Util/src/dorkbox/util/storage/DiskStorageIfface.java
Normal file
177
Dorkbox-Util/src/dorkbox/util/storage/DiskStorageIfface.java
Normal file
|
@ -0,0 +1,177 @@
|
||||||
|
package dorkbox.util.storage;
|
||||||
|
|
||||||
|
import dorkbox.util.bytes.ByteArrayWrapper;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public
|
||||||
|
interface DiskStorageIfface {
|
||||||
|
/**
|
||||||
|
* Returns the number of objects in the database.
|
||||||
|
*/
|
||||||
|
int size();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks if there is a object corresponding to the given key.
|
||||||
|
*/
|
||||||
|
boolean contains(String key);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a object using the default (blank) key, and casts it to the expected class
|
||||||
|
*/
|
||||||
|
<T> T get();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a object using the specific key, and casts it to the expected class
|
||||||
|
*/
|
||||||
|
<T> T get(String key);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a object using the specific key, and casts it to the expected class
|
||||||
|
*/
|
||||||
|
<T> T get(byte[] key);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a object using the specific key, and casts it to the expected class
|
||||||
|
*/
|
||||||
|
<T> T get(ByteArrayWrapper key);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Uses the DEFAULT key ("") to return saved data.
|
||||||
|
* <p/>
|
||||||
|
* This will check to see if there is an associated key for that data, if not - it will use data as the default
|
||||||
|
*
|
||||||
|
* @param data The data that will hold the copy of the data from disk
|
||||||
|
*/
|
||||||
|
<T> T load(T data) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the saved data for the specified key.
|
||||||
|
*
|
||||||
|
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||||
|
*/
|
||||||
|
<T> T load(String key, T data) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the saved data for the specified key.
|
||||||
|
*
|
||||||
|
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||||
|
*/
|
||||||
|
<T> T load(byte[] key, T data) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the saved data for the specified key.
|
||||||
|
*
|
||||||
|
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
<T> T load(ByteArrayWrapper key, T data) throws IOException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Saves the given data to storage with the associated key.
|
||||||
|
* <p/>
|
||||||
|
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||||
|
* deleting the old data and adding the new.
|
||||||
|
*/
|
||||||
|
void put(String key, Object data);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Saves the given data to storage with the associated key.
|
||||||
|
* <p/>
|
||||||
|
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||||
|
* deleting the old data and adding the new.
|
||||||
|
*/
|
||||||
|
void put(byte[] key, Object data);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Saves the given data to storage with the associated key.
|
||||||
|
* <p/>
|
||||||
|
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||||
|
* deleting the old data and adding the new.
|
||||||
|
*/
|
||||||
|
void put(ByteArrayWrapper key, Object data);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Adds the given object to the storage using a default (blank) key, OR -- if it has been registered, using it's registered key
|
||||||
|
* <p/>
|
||||||
|
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||||
|
* deleting the old data and adding the new.
|
||||||
|
*/
|
||||||
|
void put(Object data);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deletes an object from storage. To ALSO remove from the cache, use unRegister(key)
|
||||||
|
*
|
||||||
|
* @return true if the delete was successful. False if there were problems deleting the data.
|
||||||
|
*/
|
||||||
|
boolean delete(String key);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return the file that backs this storage
|
||||||
|
*/
|
||||||
|
File getFile();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the backing file size.
|
||||||
|
*
|
||||||
|
* @return -1 if there was an error
|
||||||
|
*/
|
||||||
|
long getFileSize();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return true if there are objects queued to be written?
|
||||||
|
*/
|
||||||
|
boolean hasWriteWaiting();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return the delay in milliseconds this will wait after the last action to flush the data to the disk
|
||||||
|
*/
|
||||||
|
long getSaveDelay();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param milliSeconds milliseconds to wait
|
||||||
|
*/
|
||||||
|
void setSaveDelay(long milliSeconds);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return the version of data stored in the database
|
||||||
|
*/
|
||||||
|
int getVersion();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sets the version of data stored in the database
|
||||||
|
*/
|
||||||
|
void setVersion(int version);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Save the storage to disk, immediately.
|
||||||
|
* <p/>
|
||||||
|
* This will save the ALL of the pending save actions to the file
|
||||||
|
*/
|
||||||
|
void commit();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Save the storage to disk, immediately.
|
||||||
|
* <p/>
|
||||||
|
* This will save the ALL of the pending save actions to the file
|
||||||
|
*/
|
||||||
|
void commit(String key, Object object);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Save the storage to disk, immediately.
|
||||||
|
* <p/>
|
||||||
|
* This will save the ALL of the pending save actions to the file
|
||||||
|
*/
|
||||||
|
void commit(byte[] key, Object object);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Save the storage to disk, immediately.
|
||||||
|
* <p/>
|
||||||
|
* This will save the ALL of the pending save actions to the file
|
||||||
|
*/
|
||||||
|
void commit(ByteArrayWrapper key, Object object);
|
||||||
|
}
|
176
Dorkbox-Util/src/dorkbox/util/storage/MakeStorage.java
Normal file
176
Dorkbox-Util/src/dorkbox/util/storage/MakeStorage.java
Normal file
|
@ -0,0 +1,176 @@
|
||||||
|
package dorkbox.util.storage;
|
||||||
|
|
||||||
|
import dorkbox.util.SerializationManager;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
public
|
||||||
|
class MakeStorage {
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(DiskStorage.class);
|
||||||
|
@SuppressWarnings("SpellCheckingInspection")
|
||||||
|
private static final Map<File, DiskStorageIfface> storages = new HashMap<File, DiskStorageIfface>(1);
|
||||||
|
|
||||||
|
public static
|
||||||
|
DiskMaker Disk() {
|
||||||
|
return new DiskMaker();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static
|
||||||
|
MemoryMaker Memory() {
|
||||||
|
return new MemoryMaker();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Closes the storage.
|
||||||
|
*/
|
||||||
|
public static
|
||||||
|
void close(File file) {
|
||||||
|
synchronized (storages) {
|
||||||
|
DiskStorageIfface storage = storages.get(file);
|
||||||
|
if (storage != null) {
|
||||||
|
if (storage instanceof DiskStorage) {
|
||||||
|
final DiskStorage diskStorage = (DiskStorage) storage;
|
||||||
|
boolean isLastOne = diskStorage.decrementReference();
|
||||||
|
if (isLastOne) {
|
||||||
|
diskStorage.close();
|
||||||
|
storages.remove(file);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Closes the storage.
|
||||||
|
*/
|
||||||
|
public static
|
||||||
|
void close(DiskStorageIfface _storage) {
|
||||||
|
synchronized (storages) {
|
||||||
|
File file = _storage.getFile();
|
||||||
|
DiskStorageIfface storage = storages.get(file);
|
||||||
|
if (storage != null) {
|
||||||
|
if (storage instanceof DiskStorage) {
|
||||||
|
final DiskStorage diskStorage = (DiskStorage) storage;
|
||||||
|
boolean isLastOne = diskStorage.decrementReference();
|
||||||
|
if (isLastOne) {
|
||||||
|
diskStorage.close();
|
||||||
|
storages.remove(file);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static
|
||||||
|
void shutdown() {
|
||||||
|
synchronized (storages) {
|
||||||
|
Collection<DiskStorageIfface> values = storages.values();
|
||||||
|
for (DiskStorageIfface storage : values) {
|
||||||
|
if (storage instanceof DiskStorage) {
|
||||||
|
//noinspection StatementWithEmptyBody
|
||||||
|
final DiskStorage diskStorage = (DiskStorage) storage;
|
||||||
|
while (!diskStorage.decrementReference()) {
|
||||||
|
}
|
||||||
|
diskStorage.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
storages.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static
|
||||||
|
void delete(File file) {
|
||||||
|
synchronized (storages) {
|
||||||
|
DiskStorageIfface remove = storages.remove(file);
|
||||||
|
if (remove != null && remove instanceof DiskStorage) {
|
||||||
|
((DiskStorage) remove).close();
|
||||||
|
}
|
||||||
|
//noinspection ResultOfMethodCallIgnored
|
||||||
|
file.delete();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static
|
||||||
|
void delete(DiskStorageIfface storage) {
|
||||||
|
File file = storage.getFile();
|
||||||
|
delete(file);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static
|
||||||
|
class DiskMaker {
|
||||||
|
private File file;
|
||||||
|
private SerializationManager serializationManager;
|
||||||
|
|
||||||
|
public
|
||||||
|
DiskMaker file(File file) {
|
||||||
|
this.file = file;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public
|
||||||
|
DiskMaker serializer(SerializationManager serializationManager) {
|
||||||
|
this.serializationManager = serializationManager;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public
|
||||||
|
DiskStorageIfface make() {
|
||||||
|
if (file == null) {
|
||||||
|
throw new IllegalArgumentException("file cannot be null!");
|
||||||
|
}
|
||||||
|
|
||||||
|
// if we load from a NEW storage at the same location as an ALREADY EXISTING storage,
|
||||||
|
// without saving the existing storage first --- whoops!
|
||||||
|
synchronized (storages) {
|
||||||
|
DiskStorageIfface storage = storages.get(file);
|
||||||
|
|
||||||
|
if (storage != null) {
|
||||||
|
if (storage instanceof DiskStorage) {
|
||||||
|
boolean waiting = storage.hasWriteWaiting();
|
||||||
|
// we want this storage to be in a fresh state
|
||||||
|
if (waiting) {
|
||||||
|
storage.commit();
|
||||||
|
}
|
||||||
|
((DiskStorage) storage).increaseReference();
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
throw new RuntimeException("Unable to change storage types for: " + file);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
try {
|
||||||
|
storage = new DiskStorage(file, serializationManager);
|
||||||
|
storages.put(file, storage);
|
||||||
|
} catch (IOException e) {
|
||||||
|
logger.error("Unable to open storage", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return storage;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public static
|
||||||
|
class MemoryMaker {
|
||||||
|
private SerializationManager serializationManager;
|
||||||
|
|
||||||
|
public
|
||||||
|
MemoryMaker serializer(SerializationManager serializationManager) {
|
||||||
|
this.serializationManager = serializationManager;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
MemoryStorage make() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
253
Dorkbox-Util/src/dorkbox/util/storage/MemoryStorage.java
Normal file
253
Dorkbox-Util/src/dorkbox/util/storage/MemoryStorage.java
Normal file
|
@ -0,0 +1,253 @@
|
||||||
|
package dorkbox.util.storage;
|
||||||
|
|
||||||
|
import dorkbox.util.bytes.ByteArrayWrapper;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
|
||||||
|
/**
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
public
|
||||||
|
class MemoryStorage implements DiskStorageIfface {
|
||||||
|
private final ConcurrentHashMap<ByteArrayWrapper, Object> storage;
|
||||||
|
private final ByteArrayWrapper defaultKey;
|
||||||
|
private int version;
|
||||||
|
|
||||||
|
private
|
||||||
|
MemoryStorage() throws IOException {
|
||||||
|
this.storage = new ConcurrentHashMap<ByteArrayWrapper, Object>();
|
||||||
|
this.defaultKey = ByteArrayWrapper.wrap("");
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the number of objects in the database.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
int size() {
|
||||||
|
return storage.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks if there is a object corresponding to the given key.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
boolean contains(final String key) {
|
||||||
|
return storage.containsKey(ByteArrayWrapper.wrap(key));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a object using the default (blank) key, and casts it to the expected class
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
<T> T get() {
|
||||||
|
return (T) storage.get(defaultKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a object using the specific key, and casts it to the expected class
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
<T> T get(final String key) {
|
||||||
|
return get(ByteArrayWrapper.wrap(key));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a object using the specific key, and casts it to the expected class
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
<T> T get(final byte[] key) {
|
||||||
|
return get(ByteArrayWrapper.wrap(key));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads a object using the specific key, and casts it to the expected class
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
<T> T get(final ByteArrayWrapper key) {
|
||||||
|
return (T) storage.get(key);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Uses the DEFAULT key ("") to return saved data.
|
||||||
|
* <p/>
|
||||||
|
* This will check to see if there is an associated key for that data, if not - it will use data as the default
|
||||||
|
*
|
||||||
|
* @param data The data that will hold the copy of the data from disk
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
<T> T load(T data) throws IOException {
|
||||||
|
return load(this.defaultKey, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the saved data for the specified key.
|
||||||
|
*
|
||||||
|
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
<T> T load(String key, T data) throws IOException {
|
||||||
|
ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
|
||||||
|
|
||||||
|
return load(wrap, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the saved data for the specified key.
|
||||||
|
*
|
||||||
|
* @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db)
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
<T> T load(byte[] key, T data) throws IOException {
|
||||||
|
return load(ByteArrayWrapper.wrap(key), data);
|
||||||
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
<T> T load(final ByteArrayWrapper key, final T data) throws IOException {
|
||||||
|
final Object o = storage.get(key);
|
||||||
|
if (o == null) {
|
||||||
|
storage.put(key, data);
|
||||||
|
}
|
||||||
|
return (T) o;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Saves the given data to storage with the associated key.
|
||||||
|
* <p/>
|
||||||
|
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||||
|
* deleting the old data and adding the new.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void put(final String key, final Object data) {
|
||||||
|
put(ByteArrayWrapper.wrap(key), data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Saves the given data to storage with the associated key.
|
||||||
|
* <p/>
|
||||||
|
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||||
|
* deleting the old data and adding the new.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void put(final byte[] key, final Object data) {
|
||||||
|
put(ByteArrayWrapper.wrap(key), data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Saves the given data to storage with the associated key.
|
||||||
|
* <p/>
|
||||||
|
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||||
|
* deleting the old data and adding the new.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void put(final ByteArrayWrapper key, final Object object) {
|
||||||
|
storage.put(key, object);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Saves the given data to storage with the associated key.
|
||||||
|
* <p/>
|
||||||
|
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||||
|
* deleting the old data and adding the new.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void put(final Object data) {
|
||||||
|
put(defaultKey, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Deletes an object from storage. To ALSO remove from the cache, use unRegister(key)
|
||||||
|
*
|
||||||
|
* @return true if the delete was successful. False if there were problems deleting the data.
|
||||||
|
*/
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
boolean delete(final String key) {
|
||||||
|
storage.remove(ByteArrayWrapper.wrap(key));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
File getFile() {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
long getFileSize() {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
boolean hasWriteWaiting() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
long getSaveDelay() {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void setSaveDelay(final long milliSeconds) {
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public synchronized
|
||||||
|
int getVersion() {
|
||||||
|
return version;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public synchronized
|
||||||
|
void setVersion(final int version) {
|
||||||
|
this.version = version;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void commit() {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void commit(final String key, final Object object) {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void commit(final byte[] key, final Object object) {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void commit(final ByteArrayWrapper key, final Object object) {
|
||||||
|
// no-op
|
||||||
|
}
|
||||||
|
}
|
|
@ -15,6 +15,11 @@
|
||||||
*/
|
*/
|
||||||
package dorkbox.util.storage;
|
package dorkbox.util.storage;
|
||||||
|
|
||||||
|
import com.esotericsoftware.kryo.io.Input;
|
||||||
|
import com.esotericsoftware.kryo.io.Output;
|
||||||
|
import dorkbox.util.SerializationManager;
|
||||||
|
import dorkbox.util.bytes.ByteArrayWrapper;
|
||||||
|
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.io.FileOutputStream;
|
import java.io.FileOutputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
@ -24,22 +29,18 @@ import java.nio.channels.FileLock;
|
||||||
import java.util.zip.DeflaterOutputStream;
|
import java.util.zip.DeflaterOutputStream;
|
||||||
import java.util.zip.InflaterInputStream;
|
import java.util.zip.InflaterInputStream;
|
||||||
|
|
||||||
import com.esotericsoftware.kryo.Kryo;
|
@SuppressWarnings("unused")
|
||||||
import com.esotericsoftware.kryo.io.Input;
|
public
|
||||||
import com.esotericsoftware.kryo.io.Output;
|
class Metadata {
|
||||||
|
|
||||||
import dorkbox.util.bytes.ByteArrayWrapper;
|
|
||||||
|
|
||||||
public class Metadata {
|
|
||||||
// The length of a key in the index.
|
// The length of a key in the index.
|
||||||
// SHA256 is 32 bytes long.
|
// SHA256 is 32 bytes long.
|
||||||
private static final int KEY_SIZE = 32;
|
private static final int KEY_SIZE = 32;
|
||||||
|
|
||||||
// Number of bytes in the record header.
|
// Number of bytes in the record header.
|
||||||
private static final int POINTER_INFO_SIZE = 16;
|
private static final int POINTER_INFO_SIZE = 16;
|
||||||
|
|
||||||
// The total length of one index entry - the key length plus the record header length.
|
// The total length of one index entry - the key length plus the record header length.
|
||||||
static final int INDEX_ENTRY_LENGTH = KEY_SIZE + POINTER_INFO_SIZE;
|
static final int INDEX_ENTRY_LENGTH = KEY_SIZE + POINTER_INFO_SIZE;
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -60,12 +61,12 @@ public class Metadata {
|
||||||
/**
|
/**
|
||||||
* Actual number of bytes of data held in this record (4 bytes).
|
* Actual number of bytes of data held in this record (4 bytes).
|
||||||
*/
|
*/
|
||||||
volatile int dataCount;
|
volatile int dataCount;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Number of bytes of data that this record can hold (4 bytes).
|
* Number of bytes of data that this record can hold (4 bytes).
|
||||||
*/
|
*/
|
||||||
volatile int dataCapacity;
|
volatile int dataCapacity;
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -78,7 +79,8 @@ public class Metadata {
|
||||||
/**
|
/**
|
||||||
* Returns a file pointer in the index pointing to the first byte in the KEY located at the given index position.
|
* Returns a file pointer in the index pointing to the first byte in the KEY located at the given index position.
|
||||||
*/
|
*/
|
||||||
static final long getMetaDataPointer(int position) {
|
static
|
||||||
|
long getMetaDataPointer(int position) {
|
||||||
return StorageBase.FILE_HEADERS_REGION_LENGTH + INDEX_ENTRY_LENGTH * position;
|
return StorageBase.FILE_HEADERS_REGION_LENGTH + INDEX_ENTRY_LENGTH * position;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -86,17 +88,20 @@ public class Metadata {
|
||||||
* Returns a file pointer in the index pointing to the first byte in the RECORD pointer located at the given index
|
* Returns a file pointer in the index pointing to the first byte in the RECORD pointer located at the given index
|
||||||
* position.
|
* position.
|
||||||
*/
|
*/
|
||||||
static final long getDataPointer(int position) {
|
static
|
||||||
long l = Metadata.getMetaDataPointer(position) + KEY_SIZE;
|
long getDataPointer(int position) {
|
||||||
return l;
|
return Metadata.getMetaDataPointer(position) + KEY_SIZE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private Metadata(ByteArrayWrapper key) {
|
private
|
||||||
|
Metadata(ByteArrayWrapper key) {
|
||||||
this.key = key;
|
this.key = key;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** we don't know how much data there is until AFTER we write the data */
|
/**
|
||||||
|
* we don't know how much data there is until AFTER we write the data
|
||||||
|
*/
|
||||||
Metadata(ByteArrayWrapper key, int recordIndex, long dataPointer) {
|
Metadata(ByteArrayWrapper key, int recordIndex, long dataPointer) {
|
||||||
this(key, recordIndex, dataPointer, 0);
|
this(key, recordIndex, dataPointer, 0);
|
||||||
}
|
}
|
||||||
|
@ -123,11 +128,13 @@ public class Metadata {
|
||||||
/**
|
/**
|
||||||
* Reads the ith HEADER (key + metadata) from the index.
|
* Reads the ith HEADER (key + metadata) from the index.
|
||||||
*/
|
*/
|
||||||
static Metadata readHeader(RandomAccessFile file, int position) throws IOException {
|
static
|
||||||
|
Metadata readHeader(RandomAccessFile file, int position) throws IOException {
|
||||||
byte[] buf = new byte[KEY_SIZE];
|
byte[] buf = new byte[KEY_SIZE];
|
||||||
long origHeaderKeyPointer = Metadata.getMetaDataPointer(position);
|
long origHeaderKeyPointer = Metadata.getMetaDataPointer(position);
|
||||||
|
|
||||||
FileLock lock = file.getChannel().lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true);
|
FileLock lock = file.getChannel()
|
||||||
|
.lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true);
|
||||||
file.seek(origHeaderKeyPointer);
|
file.seek(origHeaderKeyPointer);
|
||||||
file.readFully(buf);
|
file.readFully(buf);
|
||||||
lock.release();
|
lock.release();
|
||||||
|
@ -136,7 +143,8 @@ public class Metadata {
|
||||||
r.indexPosition = position;
|
r.indexPosition = position;
|
||||||
|
|
||||||
long recordHeaderPointer = Metadata.getDataPointer(position);
|
long recordHeaderPointer = Metadata.getDataPointer(position);
|
||||||
lock = file.getChannel().lock(origHeaderKeyPointer, KEY_SIZE, true);
|
lock = file.getChannel()
|
||||||
|
.lock(origHeaderKeyPointer, KEY_SIZE, true);
|
||||||
file.seek(recordHeaderPointer);
|
file.seek(recordHeaderPointer);
|
||||||
r.dataPointer = file.readLong();
|
r.dataPointer = file.readLong();
|
||||||
r.dataCapacity = file.readInt();
|
r.dataCapacity = file.readInt();
|
||||||
|
@ -153,7 +161,8 @@ public class Metadata {
|
||||||
void writeMetaDataInfo(RandomAccessFile file) throws IOException {
|
void writeMetaDataInfo(RandomAccessFile file) throws IOException {
|
||||||
long recordKeyPointer = Metadata.getMetaDataPointer(this.indexPosition);
|
long recordKeyPointer = Metadata.getMetaDataPointer(this.indexPosition);
|
||||||
|
|
||||||
FileLock lock = file.getChannel().lock(recordKeyPointer, KEY_SIZE, false);
|
FileLock lock = file.getChannel()
|
||||||
|
.lock(recordKeyPointer, KEY_SIZE, false);
|
||||||
file.seek(recordKeyPointer);
|
file.seek(recordKeyPointer);
|
||||||
file.write(this.key.getBytes());
|
file.write(this.key.getBytes());
|
||||||
lock.release();
|
lock.release();
|
||||||
|
@ -162,7 +171,8 @@ public class Metadata {
|
||||||
void writeDataInfo(RandomAccessFile file) throws IOException {
|
void writeDataInfo(RandomAccessFile file) throws IOException {
|
||||||
long recordHeaderPointer = getDataPointer(this.indexPosition);
|
long recordHeaderPointer = getDataPointer(this.indexPosition);
|
||||||
|
|
||||||
FileLock lock = file.getChannel().lock(recordHeaderPointer, POINTER_INFO_SIZE, false);
|
FileLock lock = file.getChannel()
|
||||||
|
.lock(recordHeaderPointer, POINTER_INFO_SIZE, false);
|
||||||
file.seek(recordHeaderPointer);
|
file.seek(recordHeaderPointer);
|
||||||
file.writeLong(this.dataPointer);
|
file.writeLong(this.dataPointer);
|
||||||
file.writeInt(this.dataCapacity);
|
file.writeInt(this.dataCapacity);
|
||||||
|
@ -177,13 +187,17 @@ public class Metadata {
|
||||||
byte[] buf = new byte[KEY_SIZE];
|
byte[] buf = new byte[KEY_SIZE];
|
||||||
|
|
||||||
long origHeaderKeyPointer = Metadata.getMetaDataPointer(this.indexPosition);
|
long origHeaderKeyPointer = Metadata.getMetaDataPointer(this.indexPosition);
|
||||||
FileLock lock = file.getChannel().lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true);
|
FileLock lock = file.getChannel()
|
||||||
|
.lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true);
|
||||||
|
|
||||||
file.seek(origHeaderKeyPointer);
|
file.seek(origHeaderKeyPointer);
|
||||||
file.readFully(buf);
|
file.readFully(buf);
|
||||||
lock.release();
|
lock.release();
|
||||||
|
|
||||||
long newHeaderKeyPointer = Metadata.getMetaDataPointer(newIndex);
|
long newHeaderKeyPointer = Metadata.getMetaDataPointer(newIndex);
|
||||||
lock = file.getChannel().lock(newHeaderKeyPointer, INDEX_ENTRY_LENGTH, false);
|
lock = file.getChannel()
|
||||||
|
.lock(newHeaderKeyPointer, INDEX_ENTRY_LENGTH, false);
|
||||||
|
|
||||||
file.seek(newHeaderKeyPointer);
|
file.seek(newHeaderKeyPointer);
|
||||||
file.write(buf);
|
file.write(buf);
|
||||||
lock.release();
|
lock.release();
|
||||||
|
@ -205,7 +219,8 @@ public class Metadata {
|
||||||
this.dataPointer = position;
|
this.dataPointer = position;
|
||||||
this.dataCapacity = this.dataCount;
|
this.dataCapacity = this.dataCount;
|
||||||
|
|
||||||
FileLock lock = file.getChannel().lock(position, this.dataCount, false);
|
FileLock lock = file.getChannel()
|
||||||
|
.lock(position, this.dataCount, false);
|
||||||
|
|
||||||
// update the file size
|
// update the file size
|
||||||
file.setLength(position + this.dataCount);
|
file.setLength(position + this.dataCount);
|
||||||
|
@ -230,7 +245,8 @@ public class Metadata {
|
||||||
byte[] buf = new byte[this.dataCount];
|
byte[] buf = new byte[this.dataCount];
|
||||||
// System.err.print("Reading data: " + this.indexPosition + " @ " + this.dataPointer + "-" + (this.dataPointer+this.dataCount) + " -- ");
|
// System.err.print("Reading data: " + this.indexPosition + " @ " + this.dataPointer + "-" + (this.dataPointer+this.dataCount) + " -- ");
|
||||||
|
|
||||||
FileLock lock = file.getChannel().lock(this.dataPointer, this.dataCount, true);
|
FileLock lock = file.getChannel()
|
||||||
|
.lock(this.dataPointer, this.dataCount, true);
|
||||||
file.seek(this.dataPointer);
|
file.seek(this.dataPointer);
|
||||||
file.readFully(buf);
|
file.readFully(buf);
|
||||||
lock.release();
|
lock.release();
|
||||||
|
@ -239,24 +255,29 @@ public class Metadata {
|
||||||
return buf;
|
return buf;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Reads the record data for the given record header.
|
* Reads the record data for the given record header.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
<T> T readData(Kryo kryo, InflaterInputStream inputStream) {
|
static
|
||||||
|
<T> T readData(SerializationManager serializationManager, InflaterInputStream inputStream) {
|
||||||
Input input = new Input(inputStream, 1024); // read 1024 at a time
|
Input input = new Input(inputStream, 1024); // read 1024 at a time
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
T readObject = (T) kryo.readClassAndObject(input);
|
T readObject = (T) serializationManager.readClassAndObject(input);
|
||||||
return readObject;
|
return readObject;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Writes data to the end of the file (which is where the datapointer is at)
|
* Writes data to the end of the file (which is where the datapointer is at)
|
||||||
*/
|
*/
|
||||||
void writeDataFast(Kryo kryo, Object data, DeflaterOutputStream outputStream) throws IOException {
|
static
|
||||||
|
void writeDataFast(final SerializationManager serializationManager,
|
||||||
|
final Object data,
|
||||||
|
final RandomAccessFile file,
|
||||||
|
final DeflaterOutputStream outputStream) throws IOException {
|
||||||
// HAVE TO LOCK BEFORE THIS IS CALLED! (AND FREE AFTERWARDS!)
|
// HAVE TO LOCK BEFORE THIS IS CALLED! (AND FREE AFTERWARDS!)
|
||||||
Output output = new Output(outputStream, 1024); // write 1024 at a time
|
Output output = new Output(outputStream, 1024); // write 1024 at a time
|
||||||
kryo.writeClassAndObject(output, data);
|
serializationManager.writeClassAndObject(output, data);
|
||||||
output.flush();
|
output.flush();
|
||||||
|
|
||||||
outputStream.flush(); // sync-flush is enabled, so the output stream will finish compressing data.
|
outputStream.flush(); // sync-flush is enabled, so the output stream will finish compressing data.
|
||||||
|
@ -265,7 +286,8 @@ public class Metadata {
|
||||||
void writeData(ByteArrayOutputStream byteArrayOutputStream, RandomAccessFile file) throws IOException {
|
void writeData(ByteArrayOutputStream byteArrayOutputStream, RandomAccessFile file) throws IOException {
|
||||||
this.dataCount = byteArrayOutputStream.size();
|
this.dataCount = byteArrayOutputStream.size();
|
||||||
|
|
||||||
FileLock lock = file.getChannel().lock(this.dataPointer, this.dataCount, false);
|
FileLock lock = file.getChannel()
|
||||||
|
.lock(this.dataPointer, this.dataCount, false);
|
||||||
|
|
||||||
FileOutputStream out = new FileOutputStream(file.getFD());
|
FileOutputStream out = new FileOutputStream(file.getFD());
|
||||||
file.seek(this.dataPointer);
|
file.seek(this.dataPointer);
|
||||||
|
@ -275,8 +297,9 @@ public class Metadata {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString() {
|
public
|
||||||
return "RecordHeader [dataPointer=" + this.dataPointer + ", dataCount=" + this.dataCount + ", dataCapacity="
|
String toString() {
|
||||||
+ this.dataCapacity + ", indexPosition=" + this.indexPosition + "]";
|
return "RecordHeader [dataPointer=" + this.dataPointer + ", dataCount=" + this.dataCount + ", dataCapacity=" + this.dataCapacity +
|
||||||
|
", indexPosition=" + this.indexPosition + "]";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,568 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright 2014 dorkbox, llc
|
|
||||||
*
|
|
||||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
* you may not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing, software
|
|
||||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
* See the License for the specific language governing permissions and
|
|
||||||
* limitations under the License.
|
|
||||||
*/
|
|
||||||
package dorkbox.util.storage;
|
|
||||||
|
|
||||||
import java.io.File;
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.lang.reflect.Field;
|
|
||||||
import java.util.Collection;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Iterator;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
|
||||||
import java.util.concurrent.locks.ReentrantLock;
|
|
||||||
|
|
||||||
import org.bouncycastle.crypto.digests.SHA256Digest;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import dorkbox.util.DelayTimer;
|
|
||||||
import dorkbox.util.OS;
|
|
||||||
import dorkbox.util.bytes.ByteArrayWrapper;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Nothing spectacular about this storage -- it allows for persistent storage of objects to disk.
|
|
||||||
*/
|
|
||||||
public class Storage {
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(Storage.class);
|
|
||||||
|
|
||||||
private static Map<File, Storage> storages = new HashMap<File, Storage>(1);
|
|
||||||
|
|
||||||
public static Storage open(String file) {
|
|
||||||
if (file == null || file.isEmpty()) {
|
|
||||||
throw new IllegalArgumentException("file cannot be null or empty!");
|
|
||||||
}
|
|
||||||
return open(new File(file));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Two types of storage.
|
|
||||||
* Raw) save/load a single object to disk (better for really large files)
|
|
||||||
* Normal) save/load key/value objects to disk (better for multiple types of data in a single file)
|
|
||||||
*/
|
|
||||||
public static Storage open(File file) {
|
|
||||||
if (file == null) {
|
|
||||||
throw new IllegalArgumentException("file cannot be null!");
|
|
||||||
}
|
|
||||||
|
|
||||||
// if we load from a NEW storage at the same location as an ALREADY EXISTING storage,
|
|
||||||
// without saving the existing storage first --- whoops!
|
|
||||||
synchronized (storages) {
|
|
||||||
Storage storage = storages.get(file);
|
|
||||||
|
|
||||||
if (storage != null) {
|
|
||||||
boolean waiting = storage.hasWriteWaiting();
|
|
||||||
// we want this storage to be in a fresh state
|
|
||||||
if (waiting) {
|
|
||||||
storage.saveNow();
|
|
||||||
}
|
|
||||||
storage.increaseReference();
|
|
||||||
} else {
|
|
||||||
try {
|
|
||||||
storage = new Storage(file);
|
|
||||||
storages.put(file, storage);
|
|
||||||
} catch (IOException e) {
|
|
||||||
logger.error("Unable to open storage", e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return storage;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Closes the storage.
|
|
||||||
*/
|
|
||||||
public static void close(File file) {
|
|
||||||
synchronized (storages) {
|
|
||||||
Storage storage = storages.get(file);
|
|
||||||
if (storage != null) {
|
|
||||||
boolean isLastOne = storage.decrementReference();
|
|
||||||
if (isLastOne) {
|
|
||||||
storage.close();
|
|
||||||
storages.remove(file);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Closes the storage.
|
|
||||||
*/
|
|
||||||
public static void close(Storage _storage) {
|
|
||||||
synchronized (storages) {
|
|
||||||
File file = _storage.getFile();
|
|
||||||
Storage storage = storages.get(file);
|
|
||||||
if (storage != null) {
|
|
||||||
boolean isLastOne = storage.decrementReference();
|
|
||||||
if (isLastOne) {
|
|
||||||
storage.close();
|
|
||||||
storages.remove(file);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void shutdown() {
|
|
||||||
synchronized(storages) {
|
|
||||||
Collection<Storage> values = storages.values();
|
|
||||||
for (Storage storage : values) {
|
|
||||||
while (!storage.decrementReference()) {
|
|
||||||
}
|
|
||||||
storage.close();
|
|
||||||
}
|
|
||||||
storages.clear();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void delete(File file) {
|
|
||||||
synchronized(storages) {
|
|
||||||
Storage remove = storages.remove(file);
|
|
||||||
if (remove != null) {
|
|
||||||
remove.close();
|
|
||||||
}
|
|
||||||
file.delete();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static void delete(Storage storage) {
|
|
||||||
File file = storage.getFile();
|
|
||||||
delete(file);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private static void copyFields(Object source, Object dest) {
|
|
||||||
Class<? extends Object> sourceClass = source.getClass();
|
|
||||||
Class<? extends Object> destClass = dest.getClass();
|
|
||||||
|
|
||||||
if (sourceClass != destClass) {
|
|
||||||
throw new IllegalArgumentException("Source and Dest objects are not of the same class!");
|
|
||||||
}
|
|
||||||
|
|
||||||
// have to walk up the object hierarchy.
|
|
||||||
while (destClass != Object.class) {
|
|
||||||
Field[] destFields = destClass.getDeclaredFields();
|
|
||||||
|
|
||||||
for (Field destField : destFields) {
|
|
||||||
String name = destField.getName();
|
|
||||||
try {
|
|
||||||
Field sourceField = sourceClass.getDeclaredField(name);
|
|
||||||
destField.setAccessible(true);
|
|
||||||
sourceField.setAccessible(true);
|
|
||||||
|
|
||||||
Object sourceObj = sourceField.get(source);
|
|
||||||
|
|
||||||
if (sourceObj instanceof Map) {
|
|
||||||
Object destObj = destField.get(dest);
|
|
||||||
if (destObj == null) {
|
|
||||||
destField.set(dest, sourceObj);
|
|
||||||
} else if (destObj instanceof Map) {
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
Map<Object, Object> sourceMap = (Map<Object, Object>) sourceObj;
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
Map<Object, Object> destMap = (Map<Object, Object>) destObj;
|
|
||||||
|
|
||||||
destMap.clear();
|
|
||||||
Iterator<?> entries = sourceMap.entrySet().iterator();
|
|
||||||
while (entries.hasNext()) {
|
|
||||||
Map.Entry<?, ?> entry = (Map.Entry<?, ?>)entries.next();
|
|
||||||
Object key = entry.getKey();
|
|
||||||
Object value = entry.getValue();
|
|
||||||
destMap.put(key, value);
|
|
||||||
}
|
|
||||||
|
|
||||||
} else {
|
|
||||||
logger.error("Incompatible field type! '{}'", name);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
destField.set(dest, sourceObj);
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
logger.error("Unable to copy field: {}", name, e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
destClass = destClass.getSuperclass();
|
|
||||||
sourceClass = sourceClass.getSuperclass();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private volatile long milliSeconds = 3000L;
|
|
||||||
private volatile DelayTimer timer;
|
|
||||||
|
|
||||||
private final ByteArrayWrapper defaultKey;
|
|
||||||
private final StorageBase storage;
|
|
||||||
|
|
||||||
private AtomicInteger references = new AtomicInteger(1);
|
|
||||||
|
|
||||||
private final ReentrantLock actionLock = new ReentrantLock();
|
|
||||||
private volatile Map<ByteArrayWrapper, Object> actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
|
|
||||||
|
|
||||||
private AtomicBoolean isOpen = new AtomicBoolean(false);
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates or opens a new database file.
|
|
||||||
*/
|
|
||||||
private Storage(File storageFile) throws IOException {
|
|
||||||
this.storage = new StorageBase(storageFile);
|
|
||||||
this.defaultKey = wrap("");
|
|
||||||
|
|
||||||
this.timer = new DelayTimer("Storage Writer", false, new DelayTimer.Callback() {
|
|
||||||
@Override
|
|
||||||
public void execute() {
|
|
||||||
Map<ByteArrayWrapper, Object> actions = Storage.this.actionMap;
|
|
||||||
|
|
||||||
ReentrantLock actionLock2 = Storage.this.actionLock;
|
|
||||||
|
|
||||||
try {
|
|
||||||
actionLock2.lock();
|
|
||||||
|
|
||||||
// do a fast swap on the actionMap.
|
|
||||||
Storage.this.actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
|
|
||||||
} finally {
|
|
||||||
actionLock2.unlock();
|
|
||||||
}
|
|
||||||
|
|
||||||
Storage.this.storage.doActionThings(actions);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
this.isOpen.set(true);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Returns the number of objects in the database.
|
|
||||||
* <p>
|
|
||||||
* SLOW because this must save all data to disk first!
|
|
||||||
*/
|
|
||||||
public final int size() {
|
|
||||||
if (!this.isOpen.get()) {
|
|
||||||
throw new RuntimeException("Unable to act on closed storage");
|
|
||||||
}
|
|
||||||
|
|
||||||
// flush actions
|
|
||||||
// timer action runs on THIS thread, not timer thread
|
|
||||||
this.timer.delay(0L);
|
|
||||||
|
|
||||||
return this.storage.size();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Checks if there is a object corresponding to the given key.
|
|
||||||
*/
|
|
||||||
public final boolean contains(String key) {
|
|
||||||
if (!this.isOpen.get()) {
|
|
||||||
throw new RuntimeException("Unable to act on closed storage");
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.storage.contains(wrap(key));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Reads a object using the default (blank) key, and casts it to the expected class
|
|
||||||
*/
|
|
||||||
public final <T> T get() {
|
|
||||||
return get0(this.defaultKey);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Reads a object using the specific key, and casts it to the expected class
|
|
||||||
*/
|
|
||||||
public final <T> T get(String key) {
|
|
||||||
ByteArrayWrapper wrap = wrap(key);
|
|
||||||
return get0(wrap);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Copies the saved data (from) into the passed-in data. This just assigns all of the values from one to the other.
|
|
||||||
* <p>
|
|
||||||
* This will check to see if there is an associated key for that data, if not - it will use the default (blank)
|
|
||||||
*
|
|
||||||
* @param data The data that will hold the copy of the data from disk
|
|
||||||
*/
|
|
||||||
public void load(Object data) {
|
|
||||||
Object source = get();
|
|
||||||
|
|
||||||
if (source != null) {
|
|
||||||
Storage.copyFields(source, data);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Copies the saved data (from) into the passed-in data. This just assigns all of the values from one to the other.
|
|
||||||
* <p>
|
|
||||||
* The key/data will be associated for the lifetime of the object.
|
|
||||||
*
|
|
||||||
* @param object The object that will hold the copy of the data from disk (but not change the reference) once this is completed
|
|
||||||
*/
|
|
||||||
public void load(String key, Object object) {
|
|
||||||
ByteArrayWrapper wrap = wrap(key);
|
|
||||||
|
|
||||||
Object source = get0(wrap);
|
|
||||||
if (source != null) {
|
|
||||||
Storage.copyFields(source, object);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Reads a object from pending or from storage
|
|
||||||
*/
|
|
||||||
private final <T> T get0(ByteArrayWrapper key) {
|
|
||||||
if (!this.isOpen.get()) {
|
|
||||||
throw new RuntimeException("Unable to act on closed storage");
|
|
||||||
}
|
|
||||||
|
|
||||||
// if the object in is pending, we get it from there
|
|
||||||
try {
|
|
||||||
this.actionLock.lock();
|
|
||||||
|
|
||||||
Object object = this.actionMap.get(key);
|
|
||||||
|
|
||||||
if (object != null) {
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
T returnObject = (T) object;
|
|
||||||
return returnObject;
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
this.actionLock.unlock();
|
|
||||||
}
|
|
||||||
|
|
||||||
// not found, so we have to go find it on disk
|
|
||||||
return this.storage.get(key);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Save the storage to disk, once xxxx milli-seconds have passed.
|
|
||||||
* This is to help prevent thrashing the disk, or wearing it out on multiple, rapid, changes.
|
|
||||||
* <p>
|
|
||||||
* This will save the ALL of the pending save actions to the file
|
|
||||||
*/
|
|
||||||
public final void saveNow() {
|
|
||||||
if (!this.isOpen.get()) {
|
|
||||||
throw new RuntimeException("Unable to act on closed storage");
|
|
||||||
}
|
|
||||||
|
|
||||||
// timer action runs on THIS thread, not timer thread
|
|
||||||
this.timer.delay(0L);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Saves the given data to storage with the associated key.
|
|
||||||
* <p>
|
|
||||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
|
||||||
* deleting the old data and adding the new.
|
|
||||||
*/
|
|
||||||
public final void save(String key, Object object) {
|
|
||||||
if (!this.isOpen.get()) {
|
|
||||||
throw new RuntimeException("Unable to act on closed storage");
|
|
||||||
}
|
|
||||||
|
|
||||||
ByteArrayWrapper wrap = wrap(key);
|
|
||||||
action(wrap, object);
|
|
||||||
|
|
||||||
// timer action runs on TIMER thread, not this thread
|
|
||||||
this.timer.delay(this.milliSeconds);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Saves the given object to storage with the associated key.
|
|
||||||
* <p>
|
|
||||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
|
||||||
* deleting the old data and adding the new.
|
|
||||||
*/
|
|
||||||
public final void saveNow(String key, Object object) {
|
|
||||||
if (!this.isOpen.get()) {
|
|
||||||
throw new RuntimeException("Unable to act on closed storage");
|
|
||||||
}
|
|
||||||
|
|
||||||
ByteArrayWrapper wrap = wrap(key);
|
|
||||||
action(wrap, object);
|
|
||||||
|
|
||||||
// timer action runs on THIS thread, not timer thread
|
|
||||||
this.timer.delay(0L);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Adds the given object to the storage using a default (blank) key, OR -- if it has been registered, using it's registered key
|
|
||||||
* <p>
|
|
||||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
|
||||||
* deleting the old data and adding the new.
|
|
||||||
*/
|
|
||||||
public final void save(Object object) {
|
|
||||||
if (!this.isOpen.get()) {
|
|
||||||
throw new RuntimeException("Unable to act on closed storage");
|
|
||||||
}
|
|
||||||
|
|
||||||
action(this.defaultKey, object);
|
|
||||||
|
|
||||||
// timer action runs on TIMER thread, not this thread
|
|
||||||
this.timer.delay(this.milliSeconds);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Adds the given object to the storage using a default (blank) key
|
|
||||||
* <p>
|
|
||||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
|
||||||
* deleting the old data and adding the new.
|
|
||||||
*/
|
|
||||||
public final void saveNow(Object object) {
|
|
||||||
if (!this.isOpen.get()) {
|
|
||||||
throw new RuntimeException("Unable to act on closed storage");
|
|
||||||
}
|
|
||||||
|
|
||||||
action(this.defaultKey, object);
|
|
||||||
|
|
||||||
// timer action runs on THIS thread, not timer thread
|
|
||||||
this.timer.delay(0L);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Deletes an object from storage. To ALSO remove from the cache, use unRegister(key)
|
|
||||||
*
|
|
||||||
* @return true if the delete was successful. False if there were problems deleting the data.
|
|
||||||
*/
|
|
||||||
public final boolean delete(String key) {
|
|
||||||
if (!this.isOpen.get()) {
|
|
||||||
throw new RuntimeException("Unable to act on closed storage");
|
|
||||||
}
|
|
||||||
|
|
||||||
ByteArrayWrapper wrap = wrap(key);
|
|
||||||
|
|
||||||
// timer action runs on THIS thread, not timer thread
|
|
||||||
this.timer.delay(0L);
|
|
||||||
|
|
||||||
return this.storage.delete(wrap);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Closes the database and file.
|
|
||||||
*/
|
|
||||||
private final void close() {
|
|
||||||
// timer action runs on THIS thread, not timer thread
|
|
||||||
this.timer.delay(0L);
|
|
||||||
|
|
||||||
// have to "close" it after we run the timer!
|
|
||||||
this.isOpen.set(false);
|
|
||||||
|
|
||||||
this.storage.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @return the file that backs this storage
|
|
||||||
*/
|
|
||||||
public final File getFile() {
|
|
||||||
return this.storage.getFile();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Gets the backing file size.
|
|
||||||
*
|
|
||||||
* @return -1 if there was an error
|
|
||||||
*/
|
|
||||||
public final long getFileSize() {
|
|
||||||
// timer action runs on THIS thread, not timer thread
|
|
||||||
this.timer.delay(0L);
|
|
||||||
|
|
||||||
return this.storage.getFileSize();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @return true if there are objects queued to be written?
|
|
||||||
*/
|
|
||||||
public final boolean hasWriteWaiting() {
|
|
||||||
if (!this.isOpen.get()) {
|
|
||||||
throw new RuntimeException("Unable to act on closed storage");
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.timer.isWaiting();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @param delay milliseconds to wait
|
|
||||||
*/
|
|
||||||
public final void setSaveDelay(long milliSeconds) {
|
|
||||||
if (!this.isOpen.get()) {
|
|
||||||
throw new RuntimeException("Unable to act on closed storage");
|
|
||||||
}
|
|
||||||
|
|
||||||
this.milliSeconds = milliSeconds;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @return the delay in milliseconds this will wait after the last action to flush the data to the disk
|
|
||||||
*/
|
|
||||||
public final long getSaveDelay() {
|
|
||||||
return this.milliSeconds;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @return the version of data stored in the database
|
|
||||||
*/
|
|
||||||
public final int getVersion() {
|
|
||||||
return this.storage.getVersion();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Sets the version of data stored in the database
|
|
||||||
*/
|
|
||||||
public final void setVersion(int version) {
|
|
||||||
this.storage.setVersion(version);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
private final ByteArrayWrapper wrap(String key) {
|
|
||||||
byte[] bytes = key.getBytes(OS.UTF_8);
|
|
||||||
|
|
||||||
SHA256Digest digest = new SHA256Digest();
|
|
||||||
digest.update(bytes, 0, bytes.length);
|
|
||||||
byte[] hashBytes = new byte[digest.getDigestSize()];
|
|
||||||
digest.doFinal(hashBytes, 0);
|
|
||||||
ByteArrayWrapper wrap = ByteArrayWrapper.wrap(hashBytes);
|
|
||||||
|
|
||||||
return wrap;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void action(ByteArrayWrapper key, Object object) {
|
|
||||||
try {
|
|
||||||
this.actionLock.lock();
|
|
||||||
|
|
||||||
// push action to map
|
|
||||||
this.actionMap.put(key, object);
|
|
||||||
} finally {
|
|
||||||
this.actionLock.unlock();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private final void increaseReference() {
|
|
||||||
this.references.incrementAndGet();
|
|
||||||
}
|
|
||||||
|
|
||||||
/** return true when this is the last reference */
|
|
||||||
private final boolean decrementReference() {
|
|
||||||
return this.references.decrementAndGet() == 0;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -15,16 +15,16 @@
|
||||||
*/
|
*/
|
||||||
package dorkbox.util.storage;
|
package dorkbox.util.storage;
|
||||||
|
|
||||||
import java.io.ByteArrayOutputStream;
|
import com.esotericsoftware.kryo.KryoException;
|
||||||
import java.io.File;
|
import com.esotericsoftware.kryo.io.Output;
|
||||||
import java.io.FileInputStream;
|
import dorkbox.util.SerializationManager;
|
||||||
import java.io.FileOutputStream;
|
import dorkbox.util.bytes.ByteArrayWrapper;
|
||||||
import java.io.IOException;
|
import org.slf4j.Logger;
|
||||||
import java.io.OutputStream;
|
import org.slf4j.LoggerFactory;
|
||||||
import java.io.RandomAccessFile;
|
|
||||||
|
import java.io.*;
|
||||||
import java.lang.ref.WeakReference;
|
import java.lang.ref.WeakReference;
|
||||||
import java.nio.channels.FileLock;
|
import java.nio.channels.FileLock;
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Map.Entry;
|
import java.util.Map.Entry;
|
||||||
|
@ -35,45 +35,39 @@ import java.util.zip.DeflaterOutputStream;
|
||||||
import java.util.zip.Inflater;
|
import java.util.zip.Inflater;
|
||||||
import java.util.zip.InflaterInputStream;
|
import java.util.zip.InflaterInputStream;
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
|
|
||||||
import com.esotericsoftware.kryo.Kryo;
|
|
||||||
import com.esotericsoftware.kryo.KryoException;
|
|
||||||
import com.esotericsoftware.kryo.io.Output;
|
|
||||||
|
|
||||||
import dorkbox.util.bytes.ByteArrayWrapper;
|
|
||||||
|
|
||||||
// a note on file locking between c and java
|
// a note on file locking between c and java
|
||||||
// http://panks-dev.blogspot.de/2008/04/linux-file-locks-java-and-others.html
|
// http://panks-dev.blogspot.de/2008/04/linux-file-locks-java-and-others.html
|
||||||
// Also, file locks on linux are ADVISORY. if an app doesn't care about locks, then it can do stuff -- even if locked by another app
|
// Also, file locks on linux are ADVISORY. if an app doesn't care about locks, then it can do stuff -- even if locked by another app
|
||||||
|
|
||||||
public class StorageBase {
|
|
||||||
|
@SuppressWarnings("unused")
|
||||||
|
public
|
||||||
|
class StorageBase {
|
||||||
private final Logger logger = LoggerFactory.getLogger(getClass());
|
private final Logger logger = LoggerFactory.getLogger(getClass());
|
||||||
|
|
||||||
|
|
||||||
// File pointer to the data start pointer header.
|
// File pointer to the data start pointer header.
|
||||||
private static final long VERSION_HEADER_LOCATION = 0;
|
private static final long VERSION_HEADER_LOCATION = 0;
|
||||||
|
|
||||||
// File pointer to the num records header.
|
// File pointer to the num records header.
|
||||||
private static final long NUM_RECORDS_HEADER_LOCATION = 4;
|
private static final long NUM_RECORDS_HEADER_LOCATION = 4;
|
||||||
|
|
||||||
// File pointer to the data start pointer header.
|
// File pointer to the data start pointer header.
|
||||||
private static final long DATA_START_HEADER_LOCATION = 8;
|
private static final long DATA_START_HEADER_LOCATION = 8;
|
||||||
|
|
||||||
// Total length in bytes of the global database headers.
|
// Total length in bytes of the global database headers.
|
||||||
static final int FILE_HEADERS_REGION_LENGTH = 16;
|
static final int FILE_HEADERS_REGION_LENGTH = 16;
|
||||||
|
|
||||||
|
|
||||||
// The in-memory index (for efficiency, all of the record info is cached in memory).
|
// The in-memory index (for efficiency, all of the record info is cached in memory).
|
||||||
private final Map<ByteArrayWrapper, Metadata> memoryIndex;
|
private final Map<ByteArrayWrapper, Metadata> memoryIndex;
|
||||||
|
|
||||||
// determines how much the index will grow by
|
// determines how much the index will grow by
|
||||||
private Float weight;
|
private final Float weight;
|
||||||
|
|
||||||
// The keys are weak! When they go, the map entry is removed!
|
// The keys are weak! When they go, the map entry is removed!
|
||||||
private final ReentrantLock referenceLock = new ReentrantLock();
|
private final ReentrantLock referenceLock = new ReentrantLock();
|
||||||
private volatile Map<ByteArrayWrapper, Object> referencePendingWrite = new HashMap<ByteArrayWrapper, Object>();
|
|
||||||
|
|
||||||
|
|
||||||
private final File baseFile;
|
private final File baseFile;
|
||||||
|
@ -97,32 +91,29 @@ public class StorageBase {
|
||||||
|
|
||||||
|
|
||||||
// save references to these, so they don't have to be created/destroyed any time there is I/O
|
// save references to these, so they don't have to be created/destroyed any time there is I/O
|
||||||
private final Kryo kryo;
|
private final SerializationManager serializationManager;
|
||||||
|
|
||||||
private Deflater deflater;
|
private final Deflater deflater;
|
||||||
private DeflaterOutputStream outputStream;
|
private final DeflaterOutputStream outputStream;
|
||||||
|
|
||||||
private Inflater inflater;
|
private final Inflater inflater;
|
||||||
private InflaterInputStream inputStream;
|
private final InflaterInputStream inputStream;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates or opens a new database file.
|
* Creates or opens a new database file.
|
||||||
*/
|
*/
|
||||||
StorageBase(String filePath) throws IOException {
|
StorageBase(File filePath, SerializationManager serializationManager) throws IOException {
|
||||||
this(new File(filePath));
|
this.serializationManager = serializationManager;
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates or opens a new database file.
|
|
||||||
*/
|
|
||||||
StorageBase(File filePath) throws IOException {
|
|
||||||
this.logger.info("Opening storage file: '{}'", filePath.getAbsolutePath());
|
this.logger.info("Opening storage file: '{}'", filePath.getAbsolutePath());
|
||||||
|
|
||||||
this.baseFile = filePath;
|
this.baseFile = filePath;
|
||||||
|
|
||||||
File parentFile = this.baseFile.getParentFile();
|
File parentFile = this.baseFile.getParentFile();
|
||||||
if (parentFile != null) {
|
if (parentFile != null && !parentFile.exists()) {
|
||||||
parentFile.mkdirs();
|
if (!parentFile.mkdirs()) {
|
||||||
|
throw new IOException("Unable to create dirs for: " + filePath);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
this.file = new RandomAccessFile(this.baseFile, "rw");
|
this.file = new RandomAccessFile(this.baseFile, "rw");
|
||||||
|
@ -132,7 +123,8 @@ public class StorageBase {
|
||||||
this.databaseVersion = this.file.readInt();
|
this.databaseVersion = this.file.readInt();
|
||||||
this.numberOfRecords = this.file.readInt();
|
this.numberOfRecords = this.file.readInt();
|
||||||
this.dataPosition = this.file.readLong();
|
this.dataPosition = this.file.readLong();
|
||||||
} else {
|
}
|
||||||
|
else {
|
||||||
setVersionNumber(this.file, 0);
|
setVersionNumber(this.file, 0);
|
||||||
|
|
||||||
// always start off with 4 records
|
// always start off with 4 records
|
||||||
|
@ -149,18 +141,16 @@ public class StorageBase {
|
||||||
throw new IllegalArgumentException("Unable to parse header information from storage. Maybe it's corrupted?");
|
throw new IllegalArgumentException("Unable to parse header information from storage. Maybe it's corrupted?");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//noinspection AutoBoxing
|
||||||
this.logger.info("Storage version: {}", this.databaseVersion);
|
this.logger.info("Storage version: {}", this.databaseVersion);
|
||||||
|
|
||||||
|
|
||||||
this.kryo = new Kryo();
|
|
||||||
this.kryo.setRegistrationRequired(false);
|
|
||||||
|
|
||||||
this.deflater = new Deflater(7, true);
|
this.deflater = new Deflater(7, true);
|
||||||
this.outputStream = new DeflaterOutputStream(new FileOutputStream(this.file.getFD()), this.deflater, 65536, true);
|
this.outputStream = new DeflaterOutputStream(new FileOutputStream(this.file.getFD()), this.deflater, 65536, true);
|
||||||
|
|
||||||
this.inflater = new Inflater(true);
|
this.inflater = new Inflater(true);
|
||||||
this.inputStream = new InflaterInputStream(new FileInputStream(this.file.getFD()), this.inflater, 65536);
|
this.inputStream = new InflaterInputStream(new FileInputStream(this.file.getFD()), this.inflater, 65536);
|
||||||
|
|
||||||
|
//noinspection AutoBoxing
|
||||||
this.weight = .5F;
|
this.weight = .5F;
|
||||||
this.memoryIndex = new ConcurrentHashMap<ByteArrayWrapper, Metadata>(this.numberOfRecords);
|
this.memoryIndex = new ConcurrentHashMap<ByteArrayWrapper, Metadata>(this.numberOfRecords);
|
||||||
|
|
||||||
|
@ -183,7 +173,8 @@ public class StorageBase {
|
||||||
/**
|
/**
|
||||||
* Returns the current number of records in the database.
|
* Returns the current number of records in the database.
|
||||||
*/
|
*/
|
||||||
final int size() {
|
final
|
||||||
|
int size() {
|
||||||
// wrapper flushes first (protected by lock)
|
// wrapper flushes first (protected by lock)
|
||||||
// not protected by lock
|
// not protected by lock
|
||||||
return this.memoryIndex.size();
|
return this.memoryIndex.size();
|
||||||
|
@ -192,21 +183,19 @@ public class StorageBase {
|
||||||
/**
|
/**
|
||||||
* Checks if there is a record belonging to the given key.
|
* Checks if there is a record belonging to the given key.
|
||||||
*/
|
*/
|
||||||
final boolean contains(ByteArrayWrapper key) {
|
final
|
||||||
|
boolean contains(ByteArrayWrapper key) {
|
||||||
// protected by lock
|
// protected by lock
|
||||||
|
|
||||||
// check to see if it's in the pending ops
|
// check to see if it's in the pending ops
|
||||||
if (this.referencePendingWrite.containsKey(key)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.memoryIndex.containsKey(key);
|
return this.memoryIndex.containsKey(key);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @return an object for a specified key ONLY FROM THE REFERENCE CACHE
|
* @return an object for a specified key ONLY FROM THE REFERENCE CACHE
|
||||||
*/
|
*/
|
||||||
final <T> T getCached(ByteArrayWrapper key) {
|
final
|
||||||
|
<T> T getCached(ByteArrayWrapper key) {
|
||||||
// protected by lock
|
// protected by lock
|
||||||
|
|
||||||
Metadata meta = this.memoryIndex.get(key);
|
Metadata meta = this.memoryIndex.get(key);
|
||||||
|
@ -223,7 +212,7 @@ public class StorageBase {
|
||||||
|
|
||||||
if (ref != null) {
|
if (ref != null) {
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
T referenceObject = (T) ref.get();
|
T referenceObject = (T) ref.get();
|
||||||
return referenceObject;
|
return referenceObject;
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
|
@ -236,7 +225,8 @@ public class StorageBase {
|
||||||
/**
|
/**
|
||||||
* @return an object for a specified key form referenceCache FIRST, then from DISK
|
* @return an object for a specified key form referenceCache FIRST, then from DISK
|
||||||
*/
|
*/
|
||||||
final <T> T get(ByteArrayWrapper key) {
|
final
|
||||||
|
<T> T get(ByteArrayWrapper key) {
|
||||||
// NOT protected by lock
|
// NOT protected by lock
|
||||||
|
|
||||||
Metadata meta = this.memoryIndex.get(key);
|
Metadata meta = this.memoryIndex.get(key);
|
||||||
|
@ -253,7 +243,7 @@ public class StorageBase {
|
||||||
|
|
||||||
if (ref != null) {
|
if (ref != null) {
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
T referenceObject = (T) ref.get();
|
T referenceObject = (T) ref.get();
|
||||||
return referenceObject;
|
return referenceObject;
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
|
@ -266,7 +256,7 @@ public class StorageBase {
|
||||||
this.inflater.reset();
|
this.inflater.reset();
|
||||||
this.file.seek(meta.dataPointer);
|
this.file.seek(meta.dataPointer);
|
||||||
|
|
||||||
T readRecordData = meta.readData(this.kryo, this.inputStream);
|
T readRecordData = Metadata.readData(this.serializationManager, this.inputStream);
|
||||||
|
|
||||||
if (readRecordData != null) {
|
if (readRecordData != null) {
|
||||||
// now stuff it into our reference cache for future lookups!
|
// now stuff it into our reference cache for future lookups!
|
||||||
|
@ -281,10 +271,10 @@ public class StorageBase {
|
||||||
|
|
||||||
return readRecordData;
|
return readRecordData;
|
||||||
} catch (KryoException e) {
|
} catch (KryoException e) {
|
||||||
this.logger.error("Error while geting data from disk. Ignoring previous value.");
|
this.logger.error("Error while getting data from disk. Ignoring previous value.");
|
||||||
return null;
|
return null;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
this.logger.error("Error while geting data from disk", e);
|
this.logger.error("Error while getting data from disk", e);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -294,7 +284,8 @@ public class StorageBase {
|
||||||
*
|
*
|
||||||
* @return true if the delete was successful. False if there were problems deleting the data.
|
* @return true if the delete was successful. False if there were problems deleting the data.
|
||||||
*/
|
*/
|
||||||
final boolean delete(ByteArrayWrapper key) {
|
final
|
||||||
|
boolean delete(ByteArrayWrapper key) {
|
||||||
// pending ops flushed (protected by lock)
|
// pending ops flushed (protected by lock)
|
||||||
// not protected by lock
|
// not protected by lock
|
||||||
Metadata delRec = this.memoryIndex.get(key);
|
Metadata delRec = this.memoryIndex.get(key);
|
||||||
|
@ -312,11 +303,13 @@ public class StorageBase {
|
||||||
/**
|
/**
|
||||||
* Closes the database and file.
|
* Closes the database and file.
|
||||||
*/
|
*/
|
||||||
final void close() {
|
final
|
||||||
|
void close() {
|
||||||
// pending ops flushed (protected by lock)
|
// pending ops flushed (protected by lock)
|
||||||
// not protected by lock
|
// not protected by lock
|
||||||
try {
|
try {
|
||||||
this.file.getFD().sync();
|
this.file.getFD()
|
||||||
|
.sync();
|
||||||
this.file.close();
|
this.file.close();
|
||||||
this.memoryIndex.clear();
|
this.memoryIndex.clear();
|
||||||
|
|
||||||
|
@ -344,22 +337,22 @@ public class StorageBase {
|
||||||
/**
|
/**
|
||||||
* @return the file that backs this storage
|
* @return the file that backs this storage
|
||||||
*/
|
*/
|
||||||
final File getFile() {
|
final
|
||||||
|
File getFile() {
|
||||||
return this.baseFile;
|
return this.baseFile;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Saves the given data to storage.
|
* Saves the given data to storage.
|
||||||
* <p>
|
* <p/>
|
||||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||||
* deleting the old data and adding the new.
|
* deleting the old data and adding the new.
|
||||||
* <p>
|
* <p/>
|
||||||
* Will also save the object in a cache.
|
* Will also save the object in a cache.
|
||||||
*
|
|
||||||
* @return the metadata for the saved object
|
|
||||||
*/
|
*/
|
||||||
private final void save0(ByteArrayWrapper key, Object object, DeflaterOutputStream fileOutputStream, Deflater deflater) {
|
private
|
||||||
|
void save0(ByteArrayWrapper key, Object object, DeflaterOutputStream fileOutputStream, Deflater deflater) {
|
||||||
deflater.reset();
|
deflater.reset();
|
||||||
|
|
||||||
Metadata metaData = this.memoryIndex.get(key);
|
Metadata metaData = this.memoryIndex.get(key);
|
||||||
|
@ -370,19 +363,23 @@ public class StorageBase {
|
||||||
try {
|
try {
|
||||||
if (currentRecordCount == 1) {
|
if (currentRecordCount == 1) {
|
||||||
// if we are the ONLY one, then we can do things differently.
|
// if we are the ONLY one, then we can do things differently.
|
||||||
// just dump the data agian to disk.
|
// just dump the data again to disk.
|
||||||
FileLock lock = this.file.getChannel().lock(this.dataPosition, Long.MAX_VALUE-this.dataPosition, false); // don't know how big it is, so max value it
|
FileLock lock = this.file.getChannel()
|
||||||
|
.lock(this.dataPosition,
|
||||||
|
Long.MAX_VALUE - this.dataPosition,
|
||||||
|
false); // don't know how big it is, so max value it
|
||||||
this.file.seek(this.dataPosition); // this is the end of the file, we know this ahead-of-time
|
this.file.seek(this.dataPosition); // this is the end of the file, we know this ahead-of-time
|
||||||
metaData.writeDataFast(this.kryo, object, fileOutputStream);
|
Metadata.writeDataFast(this.serializationManager, object, file, fileOutputStream);
|
||||||
// have to re-specify the capacity and size
|
// have to re-specify the capacity and size
|
||||||
int sizeOfWrittenData = (int) (this.file.length() - this.dataPosition);
|
int sizeOfWrittenData = (int) (this.file.length() - this.dataPosition);
|
||||||
|
|
||||||
metaData.dataCapacity = sizeOfWrittenData;
|
metaData.dataCapacity = sizeOfWrittenData;
|
||||||
metaData.dataCount = sizeOfWrittenData;
|
metaData.dataCount = sizeOfWrittenData;
|
||||||
lock.release();
|
lock.release();
|
||||||
} else {
|
}
|
||||||
|
else {
|
||||||
// this is comparatively slow, since we serialize it first to get the size, then we put it in the file.
|
// this is comparatively slow, since we serialize it first to get the size, then we put it in the file.
|
||||||
ByteArrayOutputStream dataStream = getDataAsByteArray(this.kryo, object, deflater);
|
ByteArrayOutputStream dataStream = getDataAsByteArray(this.serializationManager, object, deflater);
|
||||||
|
|
||||||
int size = dataStream.size();
|
int size = dataStream.size();
|
||||||
if (size > metaData.dataCapacity) {
|
if (size > metaData.dataCapacity) {
|
||||||
|
@ -404,10 +401,11 @@ public class StorageBase {
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
this.logger.error("Error while saving data to disk", e);
|
this.logger.error("Error while saving data to disk", e);
|
||||||
}
|
}
|
||||||
} else {
|
}
|
||||||
|
else {
|
||||||
try {
|
try {
|
||||||
// try to move the read head in order
|
// try to move the read head in order
|
||||||
setRecordCount(this.file, currentRecordCount+1);
|
setRecordCount(this.file, currentRecordCount + 1);
|
||||||
|
|
||||||
// This will make sure that there is room to write a new record. This is zero indexed.
|
// This will make sure that there is room to write a new record. This is zero indexed.
|
||||||
// this will skip around if moves occur
|
// this will skip around if moves occur
|
||||||
|
@ -425,9 +423,10 @@ public class StorageBase {
|
||||||
// save out the data. Because we KNOW that we are writing this to the end of the file,
|
// save out the data. Because we KNOW that we are writing this to the end of the file,
|
||||||
// there are some tricks we can use.
|
// there are some tricks we can use.
|
||||||
|
|
||||||
FileLock lock = this.file.getChannel().lock(length, Long.MAX_VALUE-length, false); // don't know how big it is, so max value it
|
FileLock lock = this.file.getChannel()
|
||||||
|
.lock(length, Long.MAX_VALUE - length, false); // don't know how big it is, so max value it
|
||||||
this.file.seek(length); // this is the end of the file, we know this ahead-of-time
|
this.file.seek(length); // this is the end of the file, we know this ahead-of-time
|
||||||
metaData.writeDataFast(this.kryo, object, fileOutputStream);
|
Metadata.writeDataFast(this.serializationManager, object, file, fileOutputStream);
|
||||||
lock.release();
|
lock.release();
|
||||||
|
|
||||||
metaData.dataCount = deflater.getTotalOut();
|
metaData.dataCount = deflater.getTotalOut();
|
||||||
|
@ -445,7 +444,8 @@ public class StorageBase {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private ByteArrayOutputStream getDataAsByteArray(Kryo kryo, Object data, Deflater deflater) throws IOException {
|
private static
|
||||||
|
ByteArrayOutputStream getDataAsByteArray(SerializationManager kryo, Object data, Deflater deflater) throws IOException {
|
||||||
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||||
OutputStream outputStream = new DeflaterOutputStream(byteArrayOutputStream, deflater);
|
OutputStream outputStream = new DeflaterOutputStream(byteArrayOutputStream, deflater);
|
||||||
Output output = new Output(outputStream, 1024); // write 1024 at a time
|
Output output = new Output(outputStream, 1024); // write 1024 at a time
|
||||||
|
@ -484,23 +484,27 @@ public class StorageBase {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* "intelligent" move strategy.
|
* "intelligent" move strategy.
|
||||||
*
|
* <p/>
|
||||||
* we should increase by some weight (ie: .5) would increase the number of allocated
|
* we should increase by some weight (ie: .5) would increase the number of allocated
|
||||||
* record headers by 50%, instead of just incrementing them by one at a time.
|
* record headers by 50%, instead of just incrementing them by one at a time.
|
||||||
*/
|
*/
|
||||||
private final int getWeightedNewRecordCount(int numberOfRecords) {
|
private
|
||||||
int newNumberOfRecords = numberOfRecords + 1 + (int) (numberOfRecords * this.weight); // int used for rounding
|
int getWeightedNewRecordCount(int numberOfRecords) {
|
||||||
return newNumberOfRecords;
|
//noinspection AutoUnboxing
|
||||||
|
return numberOfRecords + 1 + (int) (numberOfRecords * this.weight);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private void deleteRecordData(Metadata deletedRecord, int sizeOfDataToAdd) throws IOException {
|
private
|
||||||
|
void deleteRecordData(Metadata deletedRecord, int sizeOfDataToAdd) throws IOException {
|
||||||
if (this.file.length() == deletedRecord.dataPointer + deletedRecord.dataCapacity) {
|
if (this.file.length() == deletedRecord.dataPointer + deletedRecord.dataCapacity) {
|
||||||
// shrink file since this is the last record in the file
|
// shrink file since this is the last record in the file
|
||||||
FileLock lock = this.file.getChannel().lock(deletedRecord.dataPointer, Long.MAX_VALUE-deletedRecord.dataPointer, false);
|
FileLock lock = this.file.getChannel()
|
||||||
|
.lock(deletedRecord.dataPointer, Long.MAX_VALUE - deletedRecord.dataPointer, false);
|
||||||
this.file.setLength(deletedRecord.dataPointer);
|
this.file.setLength(deletedRecord.dataPointer);
|
||||||
lock.release();
|
lock.release();
|
||||||
} else {
|
}
|
||||||
|
else {
|
||||||
// we MIGHT be the FIRST record
|
// we MIGHT be the FIRST record
|
||||||
Metadata first = index_getMetaDataFromData(this.dataPosition);
|
Metadata first = index_getMetaDataFromData(this.dataPosition);
|
||||||
if (first == deletedRecord) {
|
if (first == deletedRecord) {
|
||||||
|
@ -515,22 +519,25 @@ public class StorageBase {
|
||||||
long endIndexPointer = Metadata.getMetaDataPointer(newNumberOfRecords);
|
long endIndexPointer = Metadata.getMetaDataPointer(newNumberOfRecords);
|
||||||
|
|
||||||
long endOfDataPointer = deletedRecord.dataPointer + deletedRecord.dataCapacity;
|
long endOfDataPointer = deletedRecord.dataPointer + deletedRecord.dataCapacity;
|
||||||
long newEndOfDataPointer = endOfDataPointer-sizeOfDataToAdd;
|
long newEndOfDataPointer = endOfDataPointer - sizeOfDataToAdd;
|
||||||
|
|
||||||
if (endIndexPointer < this.dataPosition && endIndexPointer <= newEndOfDataPointer) {
|
if (endIndexPointer < this.dataPosition && endIndexPointer <= newEndOfDataPointer) {
|
||||||
// one option is to shrink the RECORD section to fit the new data
|
// one option is to shrink the RECORD section to fit the new data
|
||||||
setDataPosition(this.file, newEndOfDataPointer);
|
setDataPosition(this.file, newEndOfDataPointer);
|
||||||
} else {
|
}
|
||||||
|
else {
|
||||||
// option two is to grow the RECORD section, and put the data at the end of the file
|
// option two is to grow the RECORD section, and put the data at the end of the file
|
||||||
setDataPosition(this.file, endOfDataPointer);
|
setDataPosition(this.file, endOfDataPointer);
|
||||||
}
|
}
|
||||||
} else {
|
}
|
||||||
|
else {
|
||||||
Metadata previous = index_getMetaDataFromData(deletedRecord.dataPointer - 1);
|
Metadata previous = index_getMetaDataFromData(deletedRecord.dataPointer - 1);
|
||||||
if (previous != null) {
|
if (previous != null) {
|
||||||
// append space of deleted record onto previous record
|
// append space of deleted record onto previous record
|
||||||
previous.dataCapacity += deletedRecord.dataCapacity;
|
previous.dataCapacity += deletedRecord.dataCapacity;
|
||||||
previous.writeDataInfo(this.file);
|
previous.writeDataInfo(this.file);
|
||||||
} else {
|
}
|
||||||
|
else {
|
||||||
// because there is no "previous", that means we MIGHT be the FIRST record
|
// because there is no "previous", that means we MIGHT be the FIRST record
|
||||||
// well, we're not the first record. which one is RIGHT before us?
|
// well, we're not the first record. which one is RIGHT before us?
|
||||||
// it should be "previous", so something fucked up
|
// it should be "previous", so something fucked up
|
||||||
|
@ -540,11 +547,14 @@ public class StorageBase {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void deleteRecordIndex(ByteArrayWrapper key, Metadata deleteRecord) throws IOException {
|
private
|
||||||
|
void deleteRecordIndex(ByteArrayWrapper key, Metadata deleteRecord) throws IOException {
|
||||||
int currentNumRecords = this.memoryIndex.size();
|
int currentNumRecords = this.memoryIndex.size();
|
||||||
|
|
||||||
if (deleteRecord.indexPosition != currentNumRecords - 1) {
|
if (deleteRecord.indexPosition != currentNumRecords - 1) {
|
||||||
Metadata last = Metadata.readHeader(this.file, currentNumRecords - 1);
|
Metadata last = Metadata.readHeader(this.file, currentNumRecords - 1);
|
||||||
|
assert last != null;
|
||||||
|
|
||||||
last.move(this.file, deleteRecord.indexPosition);
|
last.move(this.file, deleteRecord.indexPosition);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -557,10 +567,12 @@ public class StorageBase {
|
||||||
/**
|
/**
|
||||||
* Writes the number of records header to the file.
|
* Writes the number of records header to the file.
|
||||||
*/
|
*/
|
||||||
private final void setVersionNumber(RandomAccessFile file, int versionNumber) throws IOException {
|
private
|
||||||
|
void setVersionNumber(RandomAccessFile file, int versionNumber) throws IOException {
|
||||||
this.databaseVersion = versionNumber;
|
this.databaseVersion = versionNumber;
|
||||||
|
|
||||||
FileLock lock = this.file.getChannel().lock(VERSION_HEADER_LOCATION, 4, false);
|
FileLock lock = this.file.getChannel()
|
||||||
|
.lock(VERSION_HEADER_LOCATION, 4, false);
|
||||||
file.seek(VERSION_HEADER_LOCATION);
|
file.seek(VERSION_HEADER_LOCATION);
|
||||||
file.writeInt(versionNumber);
|
file.writeInt(versionNumber);
|
||||||
lock.release();
|
lock.release();
|
||||||
|
@ -569,10 +581,12 @@ public class StorageBase {
|
||||||
/**
|
/**
|
||||||
* Writes the number of records header to the file.
|
* Writes the number of records header to the file.
|
||||||
*/
|
*/
|
||||||
private final void setRecordCount(RandomAccessFile file, int numberOfRecords) throws IOException {
|
private
|
||||||
|
void setRecordCount(RandomAccessFile file, int numberOfRecords) throws IOException {
|
||||||
this.numberOfRecords = numberOfRecords;
|
this.numberOfRecords = numberOfRecords;
|
||||||
|
|
||||||
FileLock lock = this.file.getChannel().lock(NUM_RECORDS_HEADER_LOCATION, 4, false);
|
FileLock lock = this.file.getChannel()
|
||||||
|
.lock(NUM_RECORDS_HEADER_LOCATION, 4, false);
|
||||||
file.seek(NUM_RECORDS_HEADER_LOCATION);
|
file.seek(NUM_RECORDS_HEADER_LOCATION);
|
||||||
file.writeInt(numberOfRecords);
|
file.writeInt(numberOfRecords);
|
||||||
lock.release();
|
lock.release();
|
||||||
|
@ -581,10 +595,12 @@ public class StorageBase {
|
||||||
/**
|
/**
|
||||||
* Writes the data start position to the file.
|
* Writes the data start position to the file.
|
||||||
*/
|
*/
|
||||||
private final void setDataPosition(RandomAccessFile file, long dataPositionPointer) throws IOException {
|
private
|
||||||
|
void setDataPosition(RandomAccessFile file, long dataPositionPointer) throws IOException {
|
||||||
this.dataPosition = dataPositionPointer;
|
this.dataPosition = dataPositionPointer;
|
||||||
|
|
||||||
FileLock lock = this.file.getChannel().lock(DATA_START_HEADER_LOCATION, 8, false);
|
FileLock lock = this.file.getChannel()
|
||||||
|
.lock(DATA_START_HEADER_LOCATION, 8, false);
|
||||||
file.seek(DATA_START_HEADER_LOCATION);
|
file.seek(DATA_START_HEADER_LOCATION);
|
||||||
file.writeLong(dataPositionPointer);
|
file.writeLong(dataPositionPointer);
|
||||||
lock.release();
|
lock.release();
|
||||||
|
@ -608,9 +624,12 @@ public class StorageBase {
|
||||||
* of the record data of the RecordHeader which is returned. Returns null if the location is not part of a record.
|
* of the record data of the RecordHeader which is returned. Returns null if the location is not part of a record.
|
||||||
* (O(n) mem accesses)
|
* (O(n) mem accesses)
|
||||||
*/
|
*/
|
||||||
private final Metadata index_getMetaDataFromData(long targetFp) {
|
private
|
||||||
Iterator<Metadata> iterator = this.memoryIndex.values().iterator();
|
Metadata index_getMetaDataFromData(long targetFp) {
|
||||||
|
Iterator<Metadata> iterator = this.memoryIndex.values()
|
||||||
|
.iterator();
|
||||||
|
|
||||||
|
//noinspection WhileLoopReplaceableByForEach
|
||||||
while (iterator.hasNext()) {
|
while (iterator.hasNext()) {
|
||||||
Metadata next = iterator.next();
|
Metadata next = iterator.next();
|
||||||
if (targetFp >= next.dataPointer && targetFp < next.dataPointer + next.dataCapacity) {
|
if (targetFp >= next.dataPointer && targetFp < next.dataPointer + next.dataCapacity) {
|
||||||
|
@ -623,11 +642,13 @@ public class StorageBase {
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Ensure index capacity. This operation makes sure the INDEX REGION is large enough to accommodate additional entries.
|
* Ensure index capacity. This operation makes sure the INDEX REGION is large enough to accommodate additional entries.
|
||||||
*/
|
*/
|
||||||
private final void ensureIndexCapacity(RandomAccessFile file) throws IOException {
|
private
|
||||||
|
void ensureIndexCapacity(RandomAccessFile file) throws IOException {
|
||||||
int numberOfRecords = this.numberOfRecords; // because we are zero indexed, this is ALSO the index where the record will START
|
int numberOfRecords = this.numberOfRecords; // because we are zero indexed, this is ALSO the index where the record will START
|
||||||
long endIndexPointer = Metadata.getMetaDataPointer(numberOfRecords + 1); // +1 because this is where that index will END (the start of the NEXT one)
|
long endIndexPointer = Metadata.getMetaDataPointer(
|
||||||
|
numberOfRecords + 1); // +1 because this is where that index will END (the start of the NEXT one)
|
||||||
|
|
||||||
// just set the data position to the end of the file, since we don't have any data yet.
|
// just set the data position to the end of the file, since we don't have any data yet.
|
||||||
if (endIndexPointer > file.length() && numberOfRecords == 0) {
|
if (endIndexPointer > file.length() && numberOfRecords == 0) {
|
||||||
|
|
|
@ -1,44 +1,118 @@
|
||||||
package dorkbox.util;
|
package dorkbox.util;
|
||||||
|
|
||||||
import static org.junit.Assert.fail;
|
import com.esotericsoftware.kryo.Kryo;
|
||||||
|
import com.esotericsoftware.kryo.Registration;
|
||||||
|
import com.esotericsoftware.kryo.Serializer;
|
||||||
|
import com.esotericsoftware.kryo.io.Input;
|
||||||
|
import com.esotericsoftware.kryo.io.Output;
|
||||||
|
import dorkbox.util.storage.DiskStorageIfface;
|
||||||
|
import dorkbox.util.storage.MakeStorage;
|
||||||
|
import io.netty.buffer.ByteBuf;
|
||||||
|
import org.junit.*;
|
||||||
|
import org.junit.runners.MethodSorters;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.OptionalDataException;
|
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
|
|
||||||
import org.junit.After;
|
|
||||||
import org.junit.Assert;
|
|
||||||
import org.junit.Before;
|
|
||||||
import org.junit.FixMethodOrder;
|
|
||||||
import org.junit.Test;
|
|
||||||
import org.junit.runners.MethodSorters;
|
|
||||||
|
|
||||||
import dorkbox.util.storage.Storage;
|
|
||||||
|
|
||||||
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
|
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
|
||||||
public class StorageTest {
|
public
|
||||||
|
class StorageTest {
|
||||||
|
|
||||||
private static final String TEST_DB = "sampleFile.records";
|
private static final File TEST_DB = new File("sampleFile.records");
|
||||||
|
private static final SerializationManager manager = new SerializationManager() {
|
||||||
|
Kryo kryo = new Kryo();
|
||||||
|
|
||||||
static void log(String s) {
|
@Override
|
||||||
|
public
|
||||||
|
boolean setReferences(final boolean references) {
|
||||||
|
kryo.setReferences(references);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void setRegistrationRequired(final boolean registrationRequired) {
|
||||||
|
kryo.setRegistrationRequired(registrationRequired);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void register(final Class<?> clazz) {
|
||||||
|
kryo.register(clazz);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void register(final Class<?> clazz, final Serializer<?> serializer) {
|
||||||
|
kryo.register(clazz, serializer);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
Registration register(final Class<?> type, final Serializer<?> serializer, final int id) {
|
||||||
|
kryo.register(type, serializer, id);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void write(final ByteBuf buffer, final Object message) {
|
||||||
|
final Output output = new Output();
|
||||||
|
writeClassAndObject(output, message);
|
||||||
|
buffer.writeBytes(output.getBuffer());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
Object read(final ByteBuf buffer, final int length) {
|
||||||
|
final Input input = new Input();
|
||||||
|
buffer.readBytes(input.getBuffer());
|
||||||
|
|
||||||
|
final Object o = readClassAndObject(input);
|
||||||
|
buffer.skipBytes(input.position());
|
||||||
|
|
||||||
|
return o;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
void writeClassAndObject(final Output output, final Object value) {
|
||||||
|
kryo.writeClassAndObject(output, value);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public
|
||||||
|
Object readClassAndObject(final Input input) {
|
||||||
|
return kryo.readClassAndObject(input);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
static
|
||||||
|
void log(String s) {
|
||||||
System.err.println(s);
|
System.err.println(s);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void deleteDB() {
|
public
|
||||||
Storage.delete(new File(TEST_DB));
|
void deleteDB() {
|
||||||
|
MakeStorage.delete(TEST_DB);
|
||||||
}
|
}
|
||||||
|
|
||||||
@After
|
@After
|
||||||
public void delete2DB() {
|
public
|
||||||
Storage.delete(new File(TEST_DB));
|
void delete2DB() {
|
||||||
|
MakeStorage.delete(TEST_DB);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testCreateDB() throws IOException {
|
public
|
||||||
Storage storage = Storage.open(TEST_DB);
|
void testCreateDB() throws IOException {
|
||||||
|
DiskStorageIfface storage = MakeStorage.Disk()
|
||||||
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
|
||||||
int numberOfRecords1 = storage.size();
|
int numberOfRecords1 = storage.size();
|
||||||
long size1 = storage.getFileSize();
|
long size1 = storage.getFileSize();
|
||||||
|
@ -46,67 +120,84 @@ public class StorageTest {
|
||||||
Assert.assertEquals("count is not correct", numberOfRecords1, 0);
|
Assert.assertEquals("count is not correct", numberOfRecords1, 0);
|
||||||
Assert.assertEquals("size is not correct", size1, 208L); // NOTE this will change based on the data size added!
|
Assert.assertEquals("size is not correct", size1, 208L); // NOTE this will change based on the data size added!
|
||||||
|
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
|
|
||||||
storage = Storage.open(TEST_DB);
|
storage = MakeStorage.Disk()
|
||||||
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
int numberOfRecords2 = storage.size();
|
int numberOfRecords2 = storage.size();
|
||||||
long size2 = storage.getFileSize();
|
long size2 = storage.getFileSize();
|
||||||
|
|
||||||
Assert.assertEquals("Record count is not the same", numberOfRecords1, numberOfRecords2);
|
Assert.assertEquals("Record count is not the same", numberOfRecords1, numberOfRecords2);
|
||||||
Assert.assertEquals("size is not the same", size1, size2);
|
Assert.assertEquals("size is not the same", size1, size2);
|
||||||
|
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testAddAsOne() throws IOException, ClassNotFoundException {
|
|
||||||
int total = 100;
|
|
||||||
|
|
||||||
try {
|
|
||||||
Storage storage = Storage.open(TEST_DB);
|
|
||||||
for (int i=0;i<total;i++) {
|
|
||||||
add(storage, i);
|
|
||||||
}
|
|
||||||
|
|
||||||
Storage.close(storage);
|
|
||||||
storage = Storage.open(TEST_DB);
|
|
||||||
for (int i=0;i<total;i++) {
|
|
||||||
String record1Data = createData(i);
|
|
||||||
String readRecord = readRecord(storage, i);
|
|
||||||
|
|
||||||
Assert.assertEquals("Object is not the same", record1Data, readRecord);
|
|
||||||
}
|
|
||||||
|
|
||||||
Storage.close(storage);
|
|
||||||
} catch (Exception e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
fail("Error!");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testAddNoKeyRecords() throws IOException, ClassNotFoundException {
|
public
|
||||||
|
void testAddAsOne() throws IOException, ClassNotFoundException {
|
||||||
int total = 100;
|
int total = 100;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
Storage storage = Storage.open(TEST_DB);
|
DiskStorageIfface storage = MakeStorage.Disk()
|
||||||
for (int i=0;i<total;i++) {
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
for (int i = 0; i < total; i++) {
|
||||||
|
add(storage, i);
|
||||||
|
}
|
||||||
|
|
||||||
|
MakeStorage.close(storage);
|
||||||
|
storage = MakeStorage.Disk()
|
||||||
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
for (int i = 0; i < total; i++) {
|
||||||
|
String record1Data = createData(i);
|
||||||
|
String readRecord = readRecord(storage, i);
|
||||||
|
|
||||||
|
Assert.assertEquals("Object is not the same", record1Data, readRecord);
|
||||||
|
}
|
||||||
|
|
||||||
|
MakeStorage.close(storage);
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
Assert.fail("Error!");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public
|
||||||
|
void testAddNoKeyRecords() throws IOException, ClassNotFoundException {
|
||||||
|
int total = 100;
|
||||||
|
|
||||||
|
try {
|
||||||
|
DiskStorageIfface storage = MakeStorage.Disk()
|
||||||
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
for (int i = 0; i < total; i++) {
|
||||||
log("adding record " + i + "...");
|
log("adding record " + i + "...");
|
||||||
String addRecord = createData(i);
|
String addRecord = createData(i);
|
||||||
storage.save(addRecord);
|
storage.put(addRecord);
|
||||||
|
|
||||||
log("reading record " + i + "...");
|
log("reading record " + i + "...");
|
||||||
String readData = storage.get();
|
String readData = storage.get();
|
||||||
|
|
||||||
Assert.assertEquals("Object is not the same", addRecord, readData);
|
Assert.assertEquals("Object is not the same", addRecord, readData);
|
||||||
}
|
}
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
|
|
||||||
storage = Storage.open(TEST_DB);
|
storage = MakeStorage.Disk()
|
||||||
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
|
||||||
String dataCheck = createData(total-1);
|
String dataCheck = createData(total - 1);
|
||||||
log("reading record " + (total-1) + "...");
|
log("reading record " + (total - 1) + "...");
|
||||||
String readData = storage.get();
|
String readData = storage.get();
|
||||||
|
|
||||||
Assert.assertEquals("Object is not the same", dataCheck, readData);
|
Assert.assertEquals("Object is not the same", dataCheck, readData);
|
||||||
|
@ -117,101 +208,123 @@ public class StorageTest {
|
||||||
Assert.assertEquals("count is not correct", numberOfRecords1, 1);
|
Assert.assertEquals("count is not correct", numberOfRecords1, 1);
|
||||||
Assert.assertEquals("size is not correct", size1, 235L); // NOTE this will change based on the data size added!
|
Assert.assertEquals("size is not correct", size1, 235L); // NOTE this will change based on the data size added!
|
||||||
|
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
fail("Error!");
|
Assert.fail("Error!");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testAddRecords_DelaySaveA() throws IOException, ClassNotFoundException {
|
public
|
||||||
|
void testAddRecords_DelaySaveA() throws IOException, ClassNotFoundException {
|
||||||
int total = 100;
|
int total = 100;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
Storage storage = Storage.open(TEST_DB);
|
DiskStorageIfface storage = MakeStorage.Disk()
|
||||||
for (int i=0;i<total;i++) {
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
for (int i = 0; i < total; i++) {
|
||||||
add(storage, i);
|
add(storage, i);
|
||||||
}
|
}
|
||||||
|
|
||||||
synchronized (Thread.currentThread()) {
|
synchronized (Thread.currentThread()) {
|
||||||
Thread.currentThread().wait(storage.getSaveDelay() + 1000L);
|
Thread.currentThread()
|
||||||
|
.wait(storage.getSaveDelay() + 1000L);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i=0;i<total;i++) {
|
for (int i = 0; i < total; i++) {
|
||||||
String record1Data = createData(i);
|
String record1Data = createData(i);
|
||||||
String readRecord = readRecord(storage, i);
|
String readRecord = readRecord(storage, i);
|
||||||
|
|
||||||
Assert.assertEquals("Object is not the same", record1Data, readRecord);
|
Assert.assertEquals("Object is not the same", record1Data, readRecord);
|
||||||
}
|
}
|
||||||
|
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
|
|
||||||
storage = Storage.open(TEST_DB);
|
storage = MakeStorage.Disk()
|
||||||
for (int i=0;i<total;i++) {
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
for (int i = 0; i < total; i++) {
|
||||||
String dataCheck = createData(i);
|
String dataCheck = createData(i);
|
||||||
String readRecord = readRecord(storage, i);
|
String readRecord = readRecord(storage, i);
|
||||||
|
|
||||||
Assert.assertEquals("Object is not the same", dataCheck, readRecord);
|
Assert.assertEquals("Object is not the same", dataCheck, readRecord);
|
||||||
}
|
}
|
||||||
|
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
fail("Error!");
|
Assert.fail("Error!");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testAddRecords_DelaySaveB() throws IOException, ClassNotFoundException {
|
public
|
||||||
|
void testAddRecords_DelaySaveB() throws IOException, ClassNotFoundException {
|
||||||
int total = 100;
|
int total = 100;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
Storage storage = Storage.open(TEST_DB);
|
DiskStorageIfface storage = MakeStorage.Disk()
|
||||||
for (int i=0;i<total;i++) {
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
for (int i = 0; i < total; i++) {
|
||||||
add(storage, i);
|
add(storage, i);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i=0;i<total;i++) {
|
for (int i = 0; i < total; i++) {
|
||||||
String record1Data = createData(i);
|
String record1Data = createData(i);
|
||||||
String readRecord = readRecord(storage, i);
|
String readRecord = readRecord(storage, i);
|
||||||
|
|
||||||
Assert.assertEquals("Object is not the same", record1Data, readRecord);
|
Assert.assertEquals("Object is not the same", record1Data, readRecord);
|
||||||
}
|
}
|
||||||
|
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
|
|
||||||
storage = Storage.open(TEST_DB);
|
storage = MakeStorage.Disk()
|
||||||
for (int i=0;i<total;i++) {
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
for (int i = 0; i < total; i++) {
|
||||||
String dataCheck = createData(i);
|
String dataCheck = createData(i);
|
||||||
String readRecord = readRecord(storage, i);
|
String readRecord = readRecord(storage, i);
|
||||||
|
|
||||||
Assert.assertEquals("Object is not the same", dataCheck, readRecord);
|
Assert.assertEquals("Object is not the same", dataCheck, readRecord);
|
||||||
}
|
}
|
||||||
|
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
fail("Error!");
|
Assert.fail("Error!");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testLoadRecords() throws IOException, ClassNotFoundException {
|
public
|
||||||
|
void testLoadRecords() throws IOException, ClassNotFoundException {
|
||||||
int total = 100;
|
int total = 100;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
Storage storage = Storage.open(TEST_DB);
|
DiskStorageIfface storage = MakeStorage.Disk()
|
||||||
for (int i=0;i<total;i++) {
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
for (int i = 0; i < total; i++) {
|
||||||
String addRecord = add(storage, i);
|
String addRecord = add(storage, i);
|
||||||
String readRecord = readRecord(storage, i);
|
String readRecord = readRecord(storage, i);
|
||||||
|
|
||||||
Assert.assertEquals("Object is not the same", addRecord, readRecord);
|
Assert.assertEquals("Object is not the same", addRecord, readRecord);
|
||||||
}
|
}
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
|
|
||||||
storage = Storage.open(TEST_DB);
|
storage = MakeStorage.Disk()
|
||||||
for (int i=0;i<total;i++) {
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
for (int i = 0; i < total; i++) {
|
||||||
String dataCheck = createData(i);
|
String dataCheck = createData(i);
|
||||||
String readRecord = readRecord(storage, i);
|
String readRecord = readRecord(storage, i);
|
||||||
|
|
||||||
|
@ -223,43 +336,53 @@ public class StorageTest {
|
||||||
String createKey = createKey(63);
|
String createKey = createKey(63);
|
||||||
makeData(data);
|
makeData(data);
|
||||||
|
|
||||||
storage.save(createKey, data);
|
storage.put(createKey, data);
|
||||||
|
|
||||||
Data data2 = new Data();
|
Data data2 = new Data();
|
||||||
storage.load(createKey, data2);
|
storage.load(createKey, data2);
|
||||||
Assert.assertEquals("Object is not the same", data, data2);
|
Assert.assertEquals("Object is not the same", data, data2);
|
||||||
|
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
storage = Storage.open(TEST_DB);
|
storage = MakeStorage.Disk()
|
||||||
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
|
||||||
data2 = new Data();
|
data2 = new Data();
|
||||||
storage.load(createKey, data2);
|
storage.load(createKey, data2);
|
||||||
Assert.assertEquals("Object is not the same", data, data2);
|
Assert.assertEquals("Object is not the same", data, data2);
|
||||||
|
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
fail("Error!");
|
Assert.fail("Error!");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testAddRecordsDelete1Record() throws IOException, ClassNotFoundException {
|
public
|
||||||
|
void testAddRecordsDelete1Record() throws IOException, ClassNotFoundException {
|
||||||
int total = 100;
|
int total = 100;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
Storage storage = Storage.open(TEST_DB);
|
DiskStorageIfface storage = MakeStorage.Disk()
|
||||||
for (int i=0;i<total;i++) {
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
for (int i = 0; i < total; i++) {
|
||||||
String addRecord = add(storage, i);
|
String addRecord = add(storage, i);
|
||||||
String readRecord = readRecord(storage, i);
|
String readRecord = readRecord(storage, i);
|
||||||
|
|
||||||
Assert.assertEquals("Object is not the same", addRecord, readRecord);
|
Assert.assertEquals("Object is not the same", addRecord, readRecord);
|
||||||
}
|
}
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
|
|
||||||
storage = Storage.open(TEST_DB);
|
storage = MakeStorage.Disk()
|
||||||
for (int i=0;i<total;i++) {
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
for (int i = 0; i < total; i++) {
|
||||||
String dataCheck = createData(i);
|
String dataCheck = createData(i);
|
||||||
String readRecord = readRecord(storage, i);
|
String readRecord = readRecord(storage, i);
|
||||||
|
|
||||||
|
@ -276,7 +399,7 @@ public class StorageTest {
|
||||||
|
|
||||||
|
|
||||||
if (storage.contains(createKey(3))) {
|
if (storage.contains(createKey(3))) {
|
||||||
fail("record NOT successfully deleted.");
|
Assert.fail("record NOT successfully deleted.");
|
||||||
}
|
}
|
||||||
|
|
||||||
// now we add 3 back
|
// now we add 3 back
|
||||||
|
@ -285,9 +408,12 @@ public class StorageTest {
|
||||||
|
|
||||||
Assert.assertEquals("Object is not the same", dataCheck, addRecord);
|
Assert.assertEquals("Object is not the same", dataCheck, addRecord);
|
||||||
|
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
|
|
||||||
storage = Storage.open(TEST_DB);
|
storage = MakeStorage.Disk()
|
||||||
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
|
||||||
// check 9 again
|
// check 9 again
|
||||||
readRecord = readRecord(storage, 9);
|
readRecord = readRecord(storage, 9);
|
||||||
|
@ -301,110 +427,134 @@ public class StorageTest {
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
fail("Error!");
|
Assert.fail("Error!");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testUpdateRecords() throws IOException, ClassNotFoundException {
|
public
|
||||||
|
void testUpdateRecords() throws IOException, ClassNotFoundException {
|
||||||
int total = 100;
|
int total = 100;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
Storage storage = Storage.open(TEST_DB);
|
DiskStorageIfface storage = MakeStorage.Disk()
|
||||||
for (int i=0;i<total;i++) {
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
for (int i = 0; i < total; i++) {
|
||||||
String addRecord = add(storage, i);
|
String addRecord = add(storage, i);
|
||||||
String readRecord = readRecord(storage, i);
|
String readRecord = readRecord(storage, i);
|
||||||
|
|
||||||
Assert.assertEquals("Object is not the same", addRecord, readRecord);
|
Assert.assertEquals("Object is not the same", addRecord, readRecord);
|
||||||
}
|
}
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
|
|
||||||
storage = Storage.open(TEST_DB);
|
storage = MakeStorage.Disk()
|
||||||
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
|
||||||
String updateRecord = updateRecord(storage, 3, createData(3) + "new");
|
String updateRecord = updateRecord(storage, 3, createData(3) + "new");
|
||||||
String readRecord = readRecord(storage, 3);
|
String readRecord = readRecord(storage, 3);
|
||||||
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
|
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
|
||||||
|
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
storage = Storage.open(TEST_DB);
|
storage = MakeStorage.Disk()
|
||||||
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
|
||||||
readRecord = readRecord(storage, 3);
|
readRecord = readRecord(storage, 3);
|
||||||
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
|
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
|
||||||
|
|
||||||
updateRecord = updateRecord(storage, 3, createData(3));
|
updateRecord = updateRecord(storage, 3, createData(3));
|
||||||
|
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
storage = Storage.open(TEST_DB);
|
storage = MakeStorage.Disk()
|
||||||
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
|
||||||
readRecord = readRecord(storage, 3);
|
readRecord = readRecord(storage, 3);
|
||||||
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
|
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
|
||||||
|
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
storage = Storage.open(TEST_DB);
|
storage = MakeStorage.Disk()
|
||||||
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
|
||||||
updateRecord = updateRecord(storage, 0, createData(0) + "new");
|
updateRecord = updateRecord(storage, 0, createData(0) + "new");
|
||||||
readRecord = readRecord(storage, 0);
|
readRecord = readRecord(storage, 0);
|
||||||
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
|
Assert.assertEquals("Object is not the same", updateRecord, readRecord);
|
||||||
|
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
fail("Error!");
|
Assert.fail("Error!");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSaveAllRecords() throws IOException, ClassNotFoundException {
|
public
|
||||||
|
void testSaveAllRecords() throws IOException, ClassNotFoundException {
|
||||||
int total = 100;
|
int total = 100;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
Storage storage = Storage.open(TEST_DB);
|
DiskStorageIfface storage = MakeStorage.Disk()
|
||||||
for (int i=0;i<total;i++) {
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
for (int i = 0; i < total; i++) {
|
||||||
Data data = new Data();
|
Data data = new Data();
|
||||||
makeData(data);
|
makeData(data);
|
||||||
String createKey = createKey(i);
|
String createKey = createKey(i);
|
||||||
|
|
||||||
storage.save(createKey, data);
|
storage.put(createKey, data);
|
||||||
}
|
}
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
|
|
||||||
Data data = new Data();
|
Data data = new Data();
|
||||||
makeData(data);
|
makeData(data);
|
||||||
|
|
||||||
storage = Storage.open(TEST_DB);
|
storage = MakeStorage.Disk()
|
||||||
for (int i=0;i<total;i++) {
|
.file(TEST_DB)
|
||||||
|
.serializer(manager)
|
||||||
|
.make();
|
||||||
|
for (int i = 0; i < total; i++) {
|
||||||
String createKey = createKey(i);
|
String createKey = createKey(i);
|
||||||
|
|
||||||
Data data2 = new Data();
|
Data data2 = new Data();
|
||||||
storage.load(createKey, data2);
|
storage.load(createKey, data2);
|
||||||
Assert.assertEquals("Object is not the same", data, data2);
|
Assert.assertEquals("Object is not the same", data, data2);
|
||||||
}
|
}
|
||||||
Storage.close(storage);
|
MakeStorage.close(storage);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
fail("Error!");
|
Assert.fail("Error!");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
private static
|
||||||
|
String createData(int number) {
|
||||||
private String createData(int number) {
|
|
||||||
return number + " data for record # " + number;
|
return number + " data for record # " + number;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String add(Storage storage, int number) throws IOException {
|
public static
|
||||||
|
String add(DiskStorageIfface storage, int number) throws IOException {
|
||||||
String record1Data = createData(number);
|
String record1Data = createData(number);
|
||||||
String record1Key = createKey(number);
|
String record1Key = createKey(number);
|
||||||
|
|
||||||
log("adding record " + number + "...");
|
log("adding record " + number + "...");
|
||||||
storage.save(record1Key, record1Data);
|
storage.put(record1Key, record1Data);
|
||||||
return record1Data;
|
return record1Data;
|
||||||
}
|
}
|
||||||
|
|
||||||
public String readRecord(Storage storage, int number) throws OptionalDataException, ClassNotFoundException, IOException {
|
public static
|
||||||
|
String readRecord(DiskStorageIfface storage, int number) throws ClassNotFoundException, IOException {
|
||||||
String record1Key = createKey(number);
|
String record1Key = createKey(number);
|
||||||
|
|
||||||
log("reading record " + number + "...");
|
log("reading record " + number + "...");
|
||||||
|
@ -414,30 +564,34 @@ public class StorageTest {
|
||||||
return readData;
|
return readData;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void deleteRecord(Storage storage, int nNumber) throws OptionalDataException, ClassNotFoundException, IOException {
|
public static
|
||||||
|
void deleteRecord(DiskStorageIfface storage, int nNumber) throws ClassNotFoundException, IOException {
|
||||||
String record1Key = createKey(nNumber);
|
String record1Key = createKey(nNumber);
|
||||||
|
|
||||||
log("deleting record " + nNumber + "...");
|
log("deleting record " + nNumber + "...");
|
||||||
storage.delete(record1Key);
|
storage.delete(record1Key);
|
||||||
}
|
}
|
||||||
|
|
||||||
private String updateRecord(Storage storage, int number, String newData) throws IOException {
|
private static
|
||||||
|
String updateRecord(DiskStorageIfface storage, int number, String newData) throws IOException {
|
||||||
String record1Key = createKey(number);
|
String record1Key = createKey(number);
|
||||||
|
|
||||||
log("updating record " + number + "...");
|
log("updating record " + number + "...");
|
||||||
storage.save(record1Key, newData);
|
storage.put(record1Key, newData);
|
||||||
|
|
||||||
return newData;
|
return newData;
|
||||||
}
|
}
|
||||||
|
|
||||||
private String createKey(int number) {
|
private static
|
||||||
|
String createKey(int number) {
|
||||||
return "foo" + number;
|
return "foo" + number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// from kryo unit test.
|
// from kryo unit test.
|
||||||
private void makeData(Data data) {
|
private static
|
||||||
StringBuilder buffer = new StringBuilder();
|
void makeData(Data data) {
|
||||||
|
StringBuilder buffer = new StringBuilder(128);
|
||||||
for (int i = 0; i < 3; i++) {
|
for (int i = 0; i < 3; i++) {
|
||||||
buffer.append('a');
|
buffer.append('a');
|
||||||
}
|
}
|
||||||
|
@ -446,8 +600,7 @@ public class StorageTest {
|
||||||
data.strings = new String[] {"ab012", "", null, "!@#$", "<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>"};
|
data.strings = new String[] {"ab012", "", null, "!@#$", "<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>"};
|
||||||
data.ints = new int[] {-1234567, 1234567, -1, 0, 1, Integer.MAX_VALUE, Integer.MIN_VALUE};
|
data.ints = new int[] {-1234567, 1234567, -1, 0, 1, Integer.MAX_VALUE, Integer.MIN_VALUE};
|
||||||
data.shorts = new short[] {-12345, 12345, -1, 0, 1, Short.MAX_VALUE, Short.MIN_VALUE};
|
data.shorts = new short[] {-12345, 12345, -1, 0, 1, Short.MAX_VALUE, Short.MIN_VALUE};
|
||||||
data.floats = new float[] {0, -0, 1, -1, 123456, -123456, 0.1f, 0.2f, -0.3f, (float)Math.PI, Float.MAX_VALUE,
|
data.floats = new float[] {0, -0, 1, -1, 123456, -123456, 0.1f, 0.2f, -0.3f, (float) Math.PI, Float.MAX_VALUE, Float.MIN_VALUE};
|
||||||
Float.MIN_VALUE};
|
|
||||||
data.doubles = new double[] {0, -0, 1, -1, 123456, -123456, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE, Double.MIN_VALUE};
|
data.doubles = new double[] {0, -0, 1, -1, 123456, -123456, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE, Double.MIN_VALUE};
|
||||||
data.longs = new long[] {0, -0, 1, -1, 123456, -123456, 99999999999l, -99999999999l, Long.MAX_VALUE, Long.MIN_VALUE};
|
data.longs = new long[] {0, -0, 1, -1, 123456, -123456, 99999999999l, -99999999999l, Long.MAX_VALUE, Long.MIN_VALUE};
|
||||||
data.bytes = new byte[] {-123, 123, -1, 0, 1, Byte.MAX_VALUE, Byte.MIN_VALUE};
|
data.bytes = new byte[] {-123, 123, -1, 0, 1, Byte.MAX_VALUE, Byte.MIN_VALUE};
|
||||||
|
@ -455,17 +608,17 @@ public class StorageTest {
|
||||||
data.booleans = new boolean[] {true, false};
|
data.booleans = new boolean[] {true, false};
|
||||||
data.Ints = new Integer[] {-1234567, 1234567, -1, 0, 1, Integer.MAX_VALUE, Integer.MIN_VALUE};
|
data.Ints = new Integer[] {-1234567, 1234567, -1, 0, 1, Integer.MAX_VALUE, Integer.MIN_VALUE};
|
||||||
data.Shorts = new Short[] {-12345, 12345, -1, 0, 1, Short.MAX_VALUE, Short.MIN_VALUE};
|
data.Shorts = new Short[] {-12345, 12345, -1, 0, 1, Short.MAX_VALUE, Short.MIN_VALUE};
|
||||||
data.Floats = new Float[] {0f, -0f, 1f, -1f, 123456f, -123456f, 0.1f, 0.2f, -0.3f, (float)Math.PI, Float.MAX_VALUE,
|
data.Floats = new Float[] {0f, -0f, 1f, -1f, 123456f, -123456f, 0.1f, 0.2f, -0.3f, (float) Math.PI, Float.MAX_VALUE,
|
||||||
Float.MIN_VALUE};
|
Float.MIN_VALUE};
|
||||||
data.Doubles = new Double[] {0d, -0d, 1d, -1d, 123456d, -123456d, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE,
|
data.Doubles = new Double[] {0d, -0d, 1d, -1d, 123456d, -123456d, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE, Double.MIN_VALUE};
|
||||||
Double.MIN_VALUE};
|
|
||||||
data.Longs = new Long[] {0l, -0l, 1l, -1l, 123456l, -123456l, 99999999999l, -99999999999l, Long.MAX_VALUE, Long.MIN_VALUE};
|
data.Longs = new Long[] {0l, -0l, 1l, -1l, 123456l, -123456l, 99999999999l, -99999999999l, Long.MAX_VALUE, Long.MIN_VALUE};
|
||||||
data.Bytes = new Byte[] {-123, 123, -1, 0, 1, Byte.MAX_VALUE, Byte.MIN_VALUE};
|
data.Bytes = new Byte[] {-123, 123, -1, 0, 1, Byte.MAX_VALUE, Byte.MIN_VALUE};
|
||||||
data.Chars = new Character[] {32345, 12345, 0, 1, 63, Character.MAX_VALUE, Character.MIN_VALUE};
|
data.Chars = new Character[] {32345, 12345, 0, 1, 63, Character.MAX_VALUE, Character.MIN_VALUE};
|
||||||
data.Booleans = new Boolean[] {true, false};
|
data.Booleans = new Boolean[] {true, false};
|
||||||
}
|
}
|
||||||
|
|
||||||
public static class Data {
|
public static
|
||||||
|
class Data {
|
||||||
public String string;
|
public String string;
|
||||||
public String[] strings;
|
public String[] strings;
|
||||||
public int[] ints;
|
public int[] ints;
|
||||||
|
@ -485,11 +638,13 @@ public class StorageTest {
|
||||||
public Character[] Chars;
|
public Character[] Chars;
|
||||||
public Boolean[] Booleans;
|
public Boolean[] Booleans;
|
||||||
|
|
||||||
public Data() {
|
public
|
||||||
|
Data() {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public int hashCode () {
|
public
|
||||||
|
int hashCode() {
|
||||||
final int prime = 31;
|
final int prime = 31;
|
||||||
int result = 1;
|
int result = 1;
|
||||||
result = prime * result + Arrays.hashCode(this.Booleans);
|
result = prime * result + Arrays.hashCode(this.Booleans);
|
||||||
|
@ -514,7 +669,8 @@ public class StorageTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public boolean equals (Object obj) {
|
public
|
||||||
|
boolean equals(Object obj) {
|
||||||
if (this == obj) {
|
if (this == obj) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
@ -524,7 +680,7 @@ public class StorageTest {
|
||||||
if (getClass() != obj.getClass()) {
|
if (getClass() != obj.getClass()) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
Data other = (Data)obj;
|
Data other = (Data) obj;
|
||||||
if (!Arrays.equals(this.Booleans, other.Booleans)) {
|
if (!Arrays.equals(this.Booleans, other.Booleans)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -577,7 +733,8 @@ public class StorageTest {
|
||||||
if (other.string != null) {
|
if (other.string != null) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
} else if (!this.string.equals(other.string)) {
|
}
|
||||||
|
else if (!this.string.equals(other.string)) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (!Arrays.equals(this.strings, other.strings)) {
|
if (!Arrays.equals(this.strings, other.strings)) {
|
||||||
|
@ -587,7 +744,8 @@ public class StorageTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public String toString () {
|
public
|
||||||
|
String toString() {
|
||||||
return "Data";
|
return "Data";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user