dorkbox/Utilities
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Storage now depends on StorageKey (instead of ByteArrayWrapper), in order

Browse Source 
to make it clear that it must be a SHA256 hash, and not just bytes of arbitrary length.
This commit is contained in:
nathan 2017-08-01 00:52:53 +02:00
parent 2a7d8397e7
commit 6decb2b234
6 changed files with 91 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/dorkbox/util/storage/DiskStorage.java
View File

@ -27,7 +27,6 @@ import org.slf4j.Logger;
import dorkbox.util.DelayTimer;
import dorkbox.util.SerializationManager;
import dorkbox.util.bytes.ByteArrayWrapper;
/**
@ -38,7 +37,7 @@ import dorkbox.util.bytes.ByteArrayWrapper;
@SuppressWarnings({"Convert2Diamond", "Convert2Lambda"})
class DiskStorage implements Storage {
private final DelayTimer timer;
private final ByteArrayWrapper defaultKey;
private final StorageKey defaultKey;
private final StorageBase storage;
private final AtomicInteger references = new AtomicInteger(1);
@ -46,14 +45,14 @@ class DiskStorage implements Storage {
private final AtomicBoolean isOpen = new AtomicBoolean(false);
private volatile long milliSeconds = 3000L;
private volatile Map<ByteArrayWrapper, Object> actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
private volatile Map<StorageKey, Object> actionMap = new ConcurrentHashMap<StorageKey, Object>();
/**
* Creates or opens a new database file.
*/
DiskStorage(File storageFile, SerializationManager serializationManager, final boolean readOnly, final Logger logger) throws IOException {
this.storage = new StorageBase(storageFile, serializationManager, logger);
this.defaultKey = ByteArrayWrapper.wrap("");
this.defaultKey = new StorageKey("");
if (readOnly) {
this.timer = null;
@ -65,12 +64,12 @@ class DiskStorage implements Storage {
void run() {
ReentrantLock actionLock2 = DiskStorage.this.actionLock;
Map<ByteArrayWrapper, Object> actions;
Map<StorageKey, Object> actions;
try {
actionLock2.lock();
// do a fast swap on the actionMap.
actions = DiskStorage.this.actionMap;
DiskStorage.this.actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
DiskStorage.this.actionMap = new ConcurrentHashMap<StorageKey, Object>();
} finally {
actionLock2.unlock();
}
@ -116,7 +115,7 @@ class DiskStorage implements Storage {
throw new RuntimeException("Unable to act on closed storage");
}
final ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
final StorageKey wrap = new StorageKey(key);
// check if our pending actions has it, or if our storage index has it
return this.actionMap.containsKey(wrap) || this.storage.contains(wrap);
}
@ -140,7 +139,7 @@ class DiskStorage implements Storage {
@Override
public final
<T> T get(String key) throws IOException {
return get0(ByteArrayWrapper.wrap(key));
return get0(new StorageKey(key));
}
/**
@ -151,7 +150,7 @@ class DiskStorage implements Storage {
@Override
public final
<T> T get(byte[] key) throws IOException {
return get0(ByteArrayWrapper.wrap(key));
return get0(new StorageKey(key));
}
/**
@ -161,7 +160,7 @@ class DiskStorage implements Storage {
*/
@Override
public final
<T> T get(ByteArrayWrapper key) throws IOException {
<T> T get(StorageKey key) throws IOException {
return get0(key);
}
@ -186,7 +185,7 @@ class DiskStorage implements Storage {
@Override
public
<T> T getAndPut(String key, T data) throws IOException {
ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
StorageKey wrap = new StorageKey(key);
return getAndPut(wrap, data);
}
@ -199,7 +198,7 @@ class DiskStorage implements Storage {
@Override
public
<T> T getAndPut(byte[] key, T data) throws IOException {
return getAndPut(ByteArrayWrapper.wrap(key), data);
return getAndPut(new StorageKey(key), data);
}
/**
@ -210,7 +209,7 @@ class DiskStorage implements Storage {
@Override
@SuppressWarnings("unchecked")
public
<T> T getAndPut(ByteArrayWrapper key, T data) throws IOException {
<T> T getAndPut(StorageKey key, T data) throws IOException {
Object source = get0(key);
if (source == null) {
@ -236,7 +235,7 @@ class DiskStorage implements Storage {
* @throws IOException if there is a problem deserializing data from disk
*/
private
<T> T get0(ByteArrayWrapper key) throws IOException {
<T> T get0(StorageKey key) throws IOException {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
@ -269,7 +268,7 @@ class DiskStorage implements Storage {
@Override
public final
void put(String key, Object object) {
put(ByteArrayWrapper.wrap(key), object);
put(new StorageKey(key), object);
}
/**
@ -281,7 +280,7 @@ class DiskStorage implements Storage {
@Override
public final
void put(byte[] key, Object object) {
put(ByteArrayWrapper.wrap(key), object);
put(new StorageKey(key), object);
}
/**
@ -292,7 +291,7 @@ class DiskStorage implements Storage {
*/
@Override
public final
void put(ByteArrayWrapper key, Object object) {
void put(StorageKey key, Object object) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
@ -338,7 +337,7 @@ class DiskStorage implements Storage {
throw new RuntimeException("Unable to act on closed storage");
}
ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
StorageKey wrap = new StorageKey(key);
// timer action runs on THIS thread, not timer thread
if (timer != null) {
@ -357,7 +356,7 @@ class DiskStorage implements Storage {
*/
@Override
public final
boolean delete(ByteArrayWrapper key) {
boolean delete(StorageKey key) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
@ -471,7 +470,7 @@ class DiskStorage implements Storage {
}
private
void action(ByteArrayWrapper key, Object object) {
void action(StorageKey key, Object object) {
try {
this.actionLock.lock();
@ -537,7 +536,7 @@ class DiskStorage implements Storage {
// timer action runs on THIS thread, not timer thread
if (timer != null) {
action(ByteArrayWrapper.wrap(key), object);
action(new StorageKey(key), object);
this.timer.delay(0L);
}
}
@ -555,7 +554,7 @@ class DiskStorage implements Storage {
}
if (timer != null) {
action(ByteArrayWrapper.wrap(key), object);
action(new StorageKey(key), object);
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
@ -570,7 +569,7 @@ class DiskStorage implements Storage {
*/
@Override
public
void putAndSave(final ByteArrayWrapper key, final Object object) {
void putAndSave(final StorageKey key, final Object object) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}

45
src/dorkbox/util/storage/MemoryStorage.java
View File

@ -19,20 +19,18 @@ import java.io.File;
import java.io.IOException;
import java.util.concurrent.ConcurrentHashMap;
import dorkbox.util.bytes.ByteArrayWrapper;
/**
* Storage that is in memory only (and is not persisted to disk)
*/
class MemoryStorage implements Storage {
private final ConcurrentHashMap<ByteArrayWrapper, Object> storage;
private final ByteArrayWrapper defaultKey;
private final ConcurrentHashMap<StorageKey, Object> storage;
private final StorageKey defaultKey;
private int version;
MemoryStorage() {
this.storage = new ConcurrentHashMap<ByteArrayWrapper, Object>();
this.defaultKey = ByteArrayWrapper.wrap("");
this.storage = new ConcurrentHashMap<StorageKey, Object>();
this.defaultKey = new StorageKey("");
}
@ -51,7 +49,7 @@ class MemoryStorage implements Storage {
@Override
public
boolean contains(final String key) {
return storage.containsKey(ByteArrayWrapper.wrap(key));
return storage.containsKey(new StorageKey(key));
}
/**
@ -70,7 +68,7 @@ class MemoryStorage implements Storage {
@Override
public
<T> T get(final String key) {
return get(ByteArrayWrapper.wrap(key));
return get(new StorageKey(key));
}
/**
@ -79,7 +77,7 @@ class MemoryStorage implements Storage {
@Override
public
<T> T get(final byte[] key) {
return get(ByteArrayWrapper.wrap(key));
return get(new StorageKey(key));
}
/**
@ -88,7 +86,7 @@ class MemoryStorage implements Storage {
@SuppressWarnings("unchecked")
@Override
public
<T> T get(final ByteArrayWrapper key) {
<T> T get(final StorageKey key) {
return (T) storage.get(key);
}
@ -113,7 +111,7 @@ class MemoryStorage implements Storage {
@Override
public
<T> T getAndPut(String key, T data) throws IOException {
ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key);
StorageKey wrap = new StorageKey(key);
return getAndPut(wrap, data);
}
@ -126,13 +124,13 @@ class MemoryStorage implements Storage {
@Override
public
<T> T getAndPut(byte[] key, T data) throws IOException {
return getAndPut(ByteArrayWrapper.wrap(key), data);
return getAndPut(new StorageKey(key), data);
}
@SuppressWarnings("unchecked")
@Override
public
<T> T getAndPut(final ByteArrayWrapper key, final T data) throws IOException {
<T> T getAndPut(final StorageKey key, final T data) throws IOException {
final Object o = storage.get(key);
if (o == null) {
storage.put(key, data);
@ -150,7 +148,7 @@ class MemoryStorage implements Storage {
@Override
public
void put(final String key, final Object data) {
put(ByteArrayWrapper.wrap(key), data);
put(new StorageKey(key), data);
}
/**
@ -162,7 +160,7 @@ class MemoryStorage implements Storage {
@Override
public
void put(final byte[] key, final Object data) {
put(ByteArrayWrapper.wrap(key), data);
put(new StorageKey(key), data);
}
/**
@ -173,7 +171,7 @@ class MemoryStorage implements Storage {
*/
@Override
public
void put(final ByteArrayWrapper key, final Object object) {
void put(final StorageKey key, final Object object) {
storage.put(key, object);
}
@ -197,7 +195,7 @@ class MemoryStorage implements Storage {
@Override
public
boolean delete(final String key) {
return delete(ByteArrayWrapper.wrap(key));
return delete(new StorageKey(key));
}
/**
@ -207,7 +205,7 @@ class MemoryStorage implements Storage {
*/
@Override
public
boolean delete(final ByteArrayWrapper key) {
boolean delete(final StorageKey key) {
storage.remove(key);
return true;
}
@ -262,18 +260,21 @@ class MemoryStorage implements Storage {
@Override
public
void putAndSave(final String key, final Object object) {
// no-op
put(key, object);
// no-save!
}
@Override
public
void putAndSave(final byte[] key, final Object object) {
// no-op
put(key, object);
// no-save!
}
@Override
public
void putAndSave(final ByteArrayWrapper key, final Object object) {
// no-op
void putAndSave(final StorageKey key, final Object object) {
put(key, object);
// no-save!
}
}

16
src/dorkbox/util/storage/Metadata.java
View File

@ -26,7 +26,6 @@ import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import dorkbox.util.SerializationManager;
import dorkbox.util.bytes.ByteArrayWrapper;
public
class Metadata {
@ -44,7 +43,7 @@ class Metadata {
/**
* This is the key to the index
*/
final ByteArrayWrapper key;
final StorageKey key;
/**
* Indicates this header's position in the file index.
@ -86,25 +85,26 @@ class Metadata {
* Returns a file pointer in the index pointing to the first byte in the RECORD pointer located at the given index
* position.
*/
static
private static
long getDataPointer(int position) {
return Metadata.getMetaDataPointer(position) + KEY_SIZE;
}
private
Metadata(ByteArrayWrapper key) {
Metadata(StorageKey key) {
this.key = key;
}
/**
* we don't know how much data there is until AFTER we write the data
*/
Metadata(ByteArrayWrapper key, int recordIndex, long dataPointer) {
Metadata(StorageKey key, int recordIndex, long dataPointer) {
this(key, recordIndex, dataPointer, 0);
}
Metadata(ByteArrayWrapper key, int recordIndex, long dataPointer, int dataCapacity) {
private
Metadata(StorageKey key, int recordIndex, long dataPointer, int dataCapacity) {
if (key.getBytes().length > KEY_SIZE) {
throw new IllegalArgumentException("Bad record key size: " + dataCapacity);
}
@ -119,6 +119,7 @@ class Metadata {
this.dataCount = dataCapacity;
}
@SuppressWarnings("unused")
int getFreeSpace() {
return this.dataCapacity - this.dataCount;
}
@ -139,7 +140,7 @@ class Metadata {
lock.release();
Metadata r = new Metadata(ByteArrayWrapper.wrap(buf));
Metadata r = new Metadata(new StorageKey(buf));
r.indexPosition = position;
long recordHeaderPointer = Metadata.getDataPointer(position);
@ -247,6 +248,7 @@ class Metadata {
/**
* Reads the record data for the given record header.
*/
private
byte[] readDataRaw(RandomAccessFile file) throws IOException {
byte[] buf = new byte[this.dataCount];

12
src/dorkbox/util/storage/Storage.java
View File

@ -15,8 +15,6 @@
*/
package dorkbox.util.storage;
import dorkbox.util.bytes.ByteArrayWrapper;
import java.io.File;
import java.io.IOException;
@ -53,7 +51,7 @@ interface Storage {
/**
* Reads a object using the specific key, and casts it to the expected class
*/
<T> T get(ByteArrayWrapper key) throws IOException;
<T> T get(StorageKey key) throws IOException;
/**
* Uses the DEFAULT key ("") to return saved data.
@ -86,7 +84,7 @@ interface Storage {
* @param key The key used to check if data already exists.
* @param data This is the default value, and if there is no value with the key in the DB this default value will be saved.
*/
<T> T getAndPut(ByteArrayWrapper key, T data) throws IOException;
<T> T getAndPut(StorageKey key, T data) throws IOException;
/**
* Saves the given data to storage with the associated key.
@ -110,7 +108,7 @@ interface Storage {
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
void put(ByteArrayWrapper key, Object data);
void put(StorageKey key, Object data);
/**
* Adds the given object to the storage using a default (blank) key, OR -- if it has been registered, using it's registered key
@ -132,7 +130,7 @@ interface Storage {
*
* @return true if the delete was successful. False if there were problems deleting the data.
*/
boolean delete(ByteArrayWrapper key);
boolean delete(StorageKey key);
/**
* @return the file that backs this storage
@ -197,5 +195,5 @@ interface Storage {
* <p/>
* This will save the ALL of the pending save actions to the file
*/
void putAndSave(ByteArrayWrapper key, Object object);
void putAndSave(StorageKey key, Object object);
}

25
src/dorkbox/util/storage/StorageBase.java
View File

@ -39,7 +39,6 @@ import com.esotericsoftware.kryo.io.Output;
import dorkbox.util.OS;
import dorkbox.util.SerializationManager;
import dorkbox.util.bytes.ByteArrayWrapper;
// a note on file locking between c and java
@ -65,7 +64,7 @@ class StorageBase {
// The in-memory index (for efficiency, all of the record info is cached in memory).
private final Map<ByteArrayWrapper, Metadata> memoryIndex;
private final Map<StorageKey, Metadata> memoryIndex;
// determines how much the index will grow by
private final Float weight;
@ -174,7 +173,7 @@ class StorageBase {
//noinspection AutoBoxing
this.weight = 0.5F;
this.memoryIndex = new ConcurrentHashMap<ByteArrayWrapper, Metadata>(this.numberOfRecords);
this.memoryIndex = new ConcurrentHashMap<StorageKey, Metadata>(this.numberOfRecords);
if (!newStorage) {
Metadata meta;
@ -210,7 +209,7 @@ class StorageBase {
* Checks if there is a record belonging to the given key.
*/
final
boolean contains(ByteArrayWrapper key) {
boolean contains(StorageKey key) {
// protected by lock
// check to see if it's in the pending ops
@ -221,7 +220,7 @@ class StorageBase {
* @return an object for a specified key ONLY FROM THE REFERENCE CACHE
*/
final
<T> T getCached(ByteArrayWrapper key) {
<T> T getCached(StorageKey key) {
// protected by lock
Metadata meta = this.memoryIndex.get(key);
@ -253,7 +252,7 @@ class StorageBase {
* @return an object for a specified key form referenceCache FIRST, then from DISK
*/
final
<T> T get(ByteArrayWrapper key) throws IOException {
<T> T get(StorageKey key) throws IOException {
// NOT protected by lock
Metadata meta = this.memoryIndex.get(key);
@ -314,7 +313,7 @@ class StorageBase {
* @return true if the delete was successful. False if there were problems deleting the data.
*/
final
boolean delete(ByteArrayWrapper key) {
boolean delete(StorageKey key) {
// pending ops flushed (protected by lock)
// not protected by lock
Metadata delRec = this.memoryIndex.get(key);
@ -398,7 +397,7 @@ class StorageBase {
* Will also save the object in a cache.
*/
private
void save0(ByteArrayWrapper key, Object object) {
void save0(StorageKey key, Object object) {
Metadata metaData = this.memoryIndex.get(key);
int currentRecordCount = this.numberOfRecords;
@ -519,15 +518,15 @@ class StorageBase {
return outputStream;
}
void doActionThings(Map<ByteArrayWrapper, Object> actions) {
void doActionThings(Map<StorageKey, Object> actions) {
// actions is thrown away after this invocation. GC can pick it up.
// we are only interested in the LAST action that happened for some data.
// items to be "autosaved" are automatically injected into "actions".
final Set<Entry<ByteArrayWrapper, Object>> entries = actions.entrySet();
for (Entry<ByteArrayWrapper, Object> entry : entries) {
final Set<Entry<StorageKey, Object>> entries = actions.entrySet();
for (Entry<StorageKey, Object> entry : entries) {
Object object = entry.getValue();
ByteArrayWrapper key = entry.getKey();
StorageKey key = entry.getKey();
// our action list is for explicitly saving objects (but not necessarily "registering" them to be auto-saved
save0(key, object);
@ -612,7 +611,7 @@ class StorageBase {
}
private
void deleteRecordIndex(ByteArrayWrapper key, Metadata deleteRecord) throws IOException {
void deleteRecordIndex(StorageKey key, Metadata deleteRecord) throws IOException {
int currentNumRecords = this.memoryIndex.size();
if (deleteRecord.indexPosition != currentNumRecords - 1) {

20
src/dorkbox/util/storage/StorageKey.java Normal file
View File

@ -0,0 +1,20 @@
package dorkbox.util.storage;
import dorkbox.util.HashUtil;
import dorkbox.util.bytes.ByteArrayWrapper;
/**
* Make a ByteArrayWrapper that is really a SHA256 hash of the bytes.
*/
public
class StorageKey extends ByteArrayWrapper {
public
StorageKey(String key) {
super(HashUtil.getSha256(key), false);
}
public
StorageKey(byte[] key) {
super(key, false);
}
}