diff --git a/Dorkbox-Util/src/dorkbox/util/storage/DiskStorage.java b/Dorkbox-Util/src/dorkbox/util/storage/DiskStorage.java new file mode 100644 index 0000000..751ef0b --- /dev/null +++ b/Dorkbox-Util/src/dorkbox/util/storage/DiskStorage.java @@ -0,0 +1,516 @@ +/* + * Copyright 2014 dorkbox, llc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package dorkbox.util.storage; + +import dorkbox.util.DelayTimer; +import dorkbox.util.SerializationManager; +import dorkbox.util.bytes.ByteArrayWrapper; + +import java.io.File; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; + + +/** + * Nothing spectacular about this storage -- it allows for persistent storage of objects to disk. + *

+ * Be wary of opening the database file in different instances. Even with file-locks, you can corrupt the data. + */ +@SuppressWarnings("unused") +public +class DiskStorage implements DiskStorageIfface { + private final DelayTimer timer; + private final ByteArrayWrapper defaultKey; + private final StorageBase storage; + + private final AtomicInteger references = new AtomicInteger(1); + private final ReentrantLock actionLock = new ReentrantLock(); + private final AtomicBoolean isOpen = new AtomicBoolean(false); + private volatile long milliSeconds = 3000L; + + private volatile Map actionMap = new ConcurrentHashMap(); + + /** + * Creates or opens a new database file. + */ + protected + DiskStorage(File storageFile, SerializationManager serializationManager) throws IOException { + this.storage = new StorageBase(storageFile, serializationManager); + this.defaultKey = ByteArrayWrapper.wrap(""); + + this.timer = new DelayTimer("Storage Writer", false, new DelayTimer.Callback() { + @Override + public + void execute() { + Map actions = DiskStorage.this.actionMap; + + ReentrantLock actionLock2 = DiskStorage.this.actionLock; + + try { + actionLock2.lock(); + + // do a fast swap on the actionMap. + DiskStorage.this.actionMap = new ConcurrentHashMap(); + } finally { + actionLock2.unlock(); + } + + DiskStorage.this.storage.doActionThings(actions); + } + }); + + this.isOpen.set(true); + } + + + + /** + * Returns the number of objects in the database. + *

+ * SLOW because this must save all data to disk first! + */ + @Override + public final + int size() { + if (!this.isOpen.get()) { + throw new RuntimeException("Unable to act on closed storage"); + } + + // flush actions + // timer action runs on THIS thread, not timer thread + this.timer.delay(0L); + + return this.storage.size(); + } + + /** + * Checks if there is a object corresponding to the given key. + */ + @Override + public final + boolean contains(String key) { + if (!this.isOpen.get()) { + throw new RuntimeException("Unable to act on closed storage"); + } + + final ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key); + // check if our pending actions has it, or if our storage index has it + return this.actionMap.containsKey(wrap) || this.storage.contains(wrap); + } + + /** + * Reads a object using the default (blank) key, and casts it to the expected class + */ + @Override + public final + T get() { + return get0(this.defaultKey); + } + + /** + * Reads a object using the specific key, and casts it to the expected class + */ + @Override + public final + T get(String key) { + return get0(ByteArrayWrapper.wrap(key)); + } + + /** + * Reads a object using the specific key, and casts it to the expected class + */ + @Override + public final + T get(byte[] key) { + return get0(ByteArrayWrapper.wrap(key)); + } + + /** + * Reads a object using the specific key, and casts it to the expected class + */ + @Override + public final + T get(ByteArrayWrapper key) { + return get0(key); + } + + /** + * Uses the DEFAULT key ("") to return saved data. + *

+ * This will check to see if there is an associated key for that data, if not - it will use data as the default + * + * @param data The data that will hold the copy of the data from disk + */ + @Override + public + T load(T data) throws IOException { + return load(this.defaultKey, data); + } + + /** + * Returns the saved data for the specified key. + * + * @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db) + */ + @Override + public + T load(String key, T data) throws IOException { + ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key); + + return load(wrap, data); + } + + /** + * Returns the saved data for the specified key. + * + * @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db) + */ + @Override + public + T load(byte[] key, T data) throws IOException { + return load(ByteArrayWrapper.wrap(key), data); + } + + /** + * Returns the saved data for the specified key. + * + * @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db) + */ + @Override + @SuppressWarnings("unchecked") + public + T load(ByteArrayWrapper key, T data) throws IOException { + Object source = get0(key); + + if (source == null) { + // returned was null, so we should take value as the default + put(key, data); + return data; + } + else { + final Class expectedClass = data.getClass(); + final Class savedCLass = source.getClass(); + + if (!expectedClass.isAssignableFrom(savedCLass)) { + throw new IOException("Saved value type '" + source.getClass() + "' is different that expected value"); + } + } + + return (T) source; + } + + /** + * Reads a object from pending or from storage + */ + private + T get0(ByteArrayWrapper key) { + if (!this.isOpen.get()) { + throw new RuntimeException("Unable to act on closed storage"); + } + + // if the object in is pending, we get it from there + try { + this.actionLock.lock(); + + Object object = this.actionMap.get(key); + + if (object != null) { + @SuppressWarnings("unchecked") + T returnObject = (T) object; + return returnObject; + } + } finally { + this.actionLock.unlock(); + } + + // not found, so we have to go find it on disk + return this.storage.get(key); + } + + /** + * Saves the given data to storage with the associated key. + *

+ * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by + * deleting the old data and adding the new. + */ + @Override + public final + void put(String key, Object object) { + put(ByteArrayWrapper.wrap(key), object); + } + + /** + * Saves the given data to storage with the associated key. + *

+ * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by + * deleting the old data and adding the new. + */ + @Override + public final + void put(byte[] key, Object object) { + put(ByteArrayWrapper.wrap(key), object); + } + + /** + * Saves the given data to storage with the associated key. + *

+ * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by + * deleting the old data and adding the new. + */ + @Override + public final + void put(ByteArrayWrapper key, Object object) { + if (!this.isOpen.get()) { + throw new RuntimeException("Unable to act on closed storage"); + } + + action(key, object); + + // timer action runs on TIMER thread, not this thread + this.timer.delay(this.milliSeconds); + } + + /** + * Adds the given object to the storage using a default (blank) key, OR -- if it has been registered, using it's registered key + *

+ * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by + * deleting the old data and adding the new. + */ + @Override + public final + void put(Object object) { + if (!this.isOpen.get()) { + throw new RuntimeException("Unable to act on closed storage"); + } + + action(this.defaultKey, object); + + // timer action runs on TIMER thread, not this thread + this.timer.delay(this.milliSeconds); + } + + /** + * Deletes an object from storage. To ALSO remove from the cache, use unRegister(key) + * + * @return true if the delete was successful. False if there were problems deleting the data. + */ + @Override + public final + boolean delete(String key) { + if (!this.isOpen.get()) { + throw new RuntimeException("Unable to act on closed storage"); + } + + ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key); + + // timer action runs on THIS thread, not timer thread + this.timer.delay(0L); + + return this.storage.delete(wrap); + } + + /** + * Closes the database and file. + */ + void close() { + // timer action runs on THIS thread, not timer thread + this.timer.delay(0L); + + // have to "close" it after we run the timer! + this.isOpen.set(false); + + this.storage.close(); + } + + /** + * @return the file that backs this storage + */ + @Override + public final + File getFile() { + return this.storage.getFile(); + } + + /** + * Gets the backing file size. + * + * @return -1 if there was an error + */ + @Override + public final + long getFileSize() { + // timer action runs on THIS thread, not timer thread + this.timer.delay(0L); + + return this.storage.getFileSize(); + } + + /** + * @return true if there are objects queued to be written? + */ + @Override + public final + boolean hasWriteWaiting() { + if (!this.isOpen.get()) { + throw new RuntimeException("Unable to act on closed storage"); + } + + return this.timer.isWaiting(); + } + + /** + * @return the delay in milliseconds this will wait after the last action to flush the data to the disk + */ + @Override + public final + long getSaveDelay() { + return this.milliSeconds; + } + + /** + * @param milliSeconds milliseconds to wait + */ + @Override + public final + void setSaveDelay(long milliSeconds) { + if (!this.isOpen.get()) { + throw new RuntimeException("Unable to act on closed storage"); + } + + this.milliSeconds = milliSeconds; + } + + /** + * @return the version of data stored in the database + */ + @Override + public final + int getVersion() { + return this.storage.getVersion(); + } + + /** + * Sets the version of data stored in the database + */ + @Override + public final + void setVersion(int version) { + this.storage.setVersion(version); + } + + private + void action(ByteArrayWrapper key, Object object) { + try { + this.actionLock.lock(); + + // push action to map + this.actionMap.put(key, object); + } finally { + this.actionLock.unlock(); + } + } + + void increaseReference() { + this.references.incrementAndGet(); + } + + /** + * return true when this is the last reference + */ + boolean decrementReference() { + return this.references.decrementAndGet() <= 0; + } + + @Override + protected + Object clone() throws CloneNotSupportedException { + return super.clone(); + } + + /** + * Save the storage to disk, immediately. + *

+ * This will save the ALL of the pending save actions to the file + */ + @Override + public final + void commit() { + if (!this.isOpen.get()) { + throw new RuntimeException("Unable to act on closed storage"); + } + + // timer action runs on THIS thread, not timer thread + this.timer.delay(0L); + } + + /** + * Save the storage to disk, immediately. + *

+ * This will save the ALL of the pending save actions to the file + */ + @Override + public + void commit(final String key, final Object object) { + if (!this.isOpen.get()) { + throw new RuntimeException("Unable to act on closed storage"); + } + + action(ByteArrayWrapper.wrap(key), object); + + // timer action runs on THIS thread, not timer thread + this.timer.delay(0L); + } + + /** + * Save the storage to disk, immediately. + *

+ * This will save the ALL of the pending save actions to the file + */ + @Override + public + void commit(final byte[] key, final Object object) { + if (!this.isOpen.get()) { + throw new RuntimeException("Unable to act on closed storage"); + } + + action(ByteArrayWrapper.wrap(key), object); + + // timer action runs on THIS thread, not timer thread + this.timer.delay(0L); + } + + /** + * Save the storage to disk, immediately. + *

+ * This will save the ALL of the pending save actions to the file + */ + @Override + public + void commit(final ByteArrayWrapper key, final Object object) { + if (!this.isOpen.get()) { + throw new RuntimeException("Unable to act on closed storage"); + } + + action(key, object); + + // timer action runs on THIS thread, not timer thread + this.timer.delay(0L); + } +} diff --git a/Dorkbox-Util/src/dorkbox/util/storage/DiskStorageIfface.java b/Dorkbox-Util/src/dorkbox/util/storage/DiskStorageIfface.java new file mode 100644 index 0000000..41330fe --- /dev/null +++ b/Dorkbox-Util/src/dorkbox/util/storage/DiskStorageIfface.java @@ -0,0 +1,177 @@ +package dorkbox.util.storage; + +import dorkbox.util.bytes.ByteArrayWrapper; + +import java.io.File; +import java.io.IOException; + +/** + * + */ +public +interface DiskStorageIfface { + /** + * Returns the number of objects in the database. + */ + int size(); + + /** + * Checks if there is a object corresponding to the given key. + */ + boolean contains(String key); + + /** + * Reads a object using the default (blank) key, and casts it to the expected class + */ + T get(); + + /** + * Reads a object using the specific key, and casts it to the expected class + */ + T get(String key); + + /** + * Reads a object using the specific key, and casts it to the expected class + */ + T get(byte[] key); + + /** + * Reads a object using the specific key, and casts it to the expected class + */ + T get(ByteArrayWrapper key); + + /** + * Uses the DEFAULT key ("") to return saved data. + *

+ * This will check to see if there is an associated key for that data, if not - it will use data as the default + * + * @param data The data that will hold the copy of the data from disk + */ + T load(T data) throws IOException; + + /** + * Returns the saved data for the specified key. + * + * @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db) + */ + T load(String key, T data) throws IOException; + + /** + * Returns the saved data for the specified key. + * + * @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db) + */ + T load(byte[] key, T data) throws IOException; + + /** + * Returns the saved data for the specified key. + * + * @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db) + */ + @SuppressWarnings("unchecked") + T load(ByteArrayWrapper key, T data) throws IOException; + + /** + * Saves the given data to storage with the associated key. + *

+ * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by + * deleting the old data and adding the new. + */ + void put(String key, Object data); + + /** + * Saves the given data to storage with the associated key. + *

+ * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by + * deleting the old data and adding the new. + */ + void put(byte[] key, Object data); + + /** + * Saves the given data to storage with the associated key. + *

+ * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by + * deleting the old data and adding the new. + */ + void put(ByteArrayWrapper key, Object data); + + /** + * Adds the given object to the storage using a default (blank) key, OR -- if it has been registered, using it's registered key + *

+ * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by + * deleting the old data and adding the new. + */ + void put(Object data); + + /** + * Deletes an object from storage. To ALSO remove from the cache, use unRegister(key) + * + * @return true if the delete was successful. False if there were problems deleting the data. + */ + boolean delete(String key); + + /** + * @return the file that backs this storage + */ + File getFile(); + + /** + * Gets the backing file size. + * + * @return -1 if there was an error + */ + long getFileSize(); + + /** + * @return true if there are objects queued to be written? + */ + boolean hasWriteWaiting(); + + /** + * @return the delay in milliseconds this will wait after the last action to flush the data to the disk + */ + long getSaveDelay(); + + /** + * @param milliSeconds milliseconds to wait + */ + void setSaveDelay(long milliSeconds); + + /** + * @return the version of data stored in the database + */ + int getVersion(); + + /** + * Sets the version of data stored in the database + */ + void setVersion(int version); + + /** + * Save the storage to disk, immediately. + *

+ * This will save the ALL of the pending save actions to the file + */ + void commit(); + + /** + * Save the storage to disk, immediately. + *

+ * This will save the ALL of the pending save actions to the file + */ + void commit(String key, Object object); + + /** + * Save the storage to disk, immediately. + *

+ * This will save the ALL of the pending save actions to the file + */ + void commit(byte[] key, Object object); + + /** + * Save the storage to disk, immediately. + *

+ * This will save the ALL of the pending save actions to the file + */ + void commit(ByteArrayWrapper key, Object object); +} diff --git a/Dorkbox-Util/src/dorkbox/util/storage/MakeStorage.java b/Dorkbox-Util/src/dorkbox/util/storage/MakeStorage.java new file mode 100644 index 0000000..cab5cf4 --- /dev/null +++ b/Dorkbox-Util/src/dorkbox/util/storage/MakeStorage.java @@ -0,0 +1,176 @@ +package dorkbox.util.storage; + +import dorkbox.util.SerializationManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +public +class MakeStorage { + private static final Logger logger = LoggerFactory.getLogger(DiskStorage.class); + @SuppressWarnings("SpellCheckingInspection") + private static final Map storages = new HashMap(1); + + public static + DiskMaker Disk() { + return new DiskMaker(); + } + + public static + MemoryMaker Memory() { + return new MemoryMaker(); + } + + /** + * Closes the storage. + */ + public static + void close(File file) { + synchronized (storages) { + DiskStorageIfface storage = storages.get(file); + if (storage != null) { + if (storage instanceof DiskStorage) { + final DiskStorage diskStorage = (DiskStorage) storage; + boolean isLastOne = diskStorage.decrementReference(); + if (isLastOne) { + diskStorage.close(); + storages.remove(file); + } + } + } + } + } + + /** + * Closes the storage. + */ + public static + void close(DiskStorageIfface _storage) { + synchronized (storages) { + File file = _storage.getFile(); + DiskStorageIfface storage = storages.get(file); + if (storage != null) { + if (storage instanceof DiskStorage) { + final DiskStorage diskStorage = (DiskStorage) storage; + boolean isLastOne = diskStorage.decrementReference(); + if (isLastOne) { + diskStorage.close(); + storages.remove(file); + } + } + } + } + } + + public static + void shutdown() { + synchronized (storages) { + Collection values = storages.values(); + for (DiskStorageIfface storage : values) { + if (storage instanceof DiskStorage) { + //noinspection StatementWithEmptyBody + final DiskStorage diskStorage = (DiskStorage) storage; + while (!diskStorage.decrementReference()) { + } + diskStorage.close(); + } + } + storages.clear(); + } + } + + public static + void delete(File file) { + synchronized (storages) { + DiskStorageIfface remove = storages.remove(file); + if (remove != null && remove instanceof DiskStorage) { + ((DiskStorage) remove).close(); + } + //noinspection ResultOfMethodCallIgnored + file.delete(); + } + } + + public static + void delete(DiskStorageIfface storage) { + File file = storage.getFile(); + delete(file); + } + + + public static + class DiskMaker { + private File file; + private SerializationManager serializationManager; + + public + DiskMaker file(File file) { + this.file = file; + return this; + } + + public + DiskMaker serializer(SerializationManager serializationManager) { + this.serializationManager = serializationManager; + return this; + } + + public + DiskStorageIfface make() { + if (file == null) { + throw new IllegalArgumentException("file cannot be null!"); + } + + // if we load from a NEW storage at the same location as an ALREADY EXISTING storage, + // without saving the existing storage first --- whoops! + synchronized (storages) { + DiskStorageIfface storage = storages.get(file); + + if (storage != null) { + if (storage instanceof DiskStorage) { + boolean waiting = storage.hasWriteWaiting(); + // we want this storage to be in a fresh state + if (waiting) { + storage.commit(); + } + ((DiskStorage) storage).increaseReference(); + } + else { + throw new RuntimeException("Unable to change storage types for: " + file); + } + } + else { + try { + storage = new DiskStorage(file, serializationManager); + storages.put(file, storage); + } catch (IOException e) { + logger.error("Unable to open storage", e); + } + } + + return storage; + } + } + } + + + public static + class MemoryMaker { + private SerializationManager serializationManager; + + public + MemoryMaker serializer(SerializationManager serializationManager) { + this.serializationManager = serializationManager; + return this; + } + + MemoryStorage make() { + return null; + } + } +} diff --git a/Dorkbox-Util/src/dorkbox/util/storage/MemoryStorage.java b/Dorkbox-Util/src/dorkbox/util/storage/MemoryStorage.java new file mode 100644 index 0000000..39054d0 --- /dev/null +++ b/Dorkbox-Util/src/dorkbox/util/storage/MemoryStorage.java @@ -0,0 +1,253 @@ +package dorkbox.util.storage; + +import dorkbox.util.bytes.ByteArrayWrapper; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; + +/** + * + */ +public +class MemoryStorage implements DiskStorageIfface { + private final ConcurrentHashMap storage; + private final ByteArrayWrapper defaultKey; + private int version; + + private + MemoryStorage() throws IOException { + this.storage = new ConcurrentHashMap(); + this.defaultKey = ByteArrayWrapper.wrap(""); + } + + + /** + * Returns the number of objects in the database. + */ + @Override + public + int size() { + return storage.size(); + } + + /** + * Checks if there is a object corresponding to the given key. + */ + @Override + public + boolean contains(final String key) { + return storage.containsKey(ByteArrayWrapper.wrap(key)); + } + + /** + * Reads a object using the default (blank) key, and casts it to the expected class + */ + @SuppressWarnings("unchecked") + @Override + public + T get() { + return (T) storage.get(defaultKey); + } + + /** + * Reads a object using the specific key, and casts it to the expected class + */ + @Override + public + T get(final String key) { + return get(ByteArrayWrapper.wrap(key)); + } + + /** + * Reads a object using the specific key, and casts it to the expected class + */ + @Override + public + T get(final byte[] key) { + return get(ByteArrayWrapper.wrap(key)); + } + + /** + * Reads a object using the specific key, and casts it to the expected class + */ + @SuppressWarnings("unchecked") + @Override + public + T get(final ByteArrayWrapper key) { + return (T) storage.get(key); + } + + /** + * Uses the DEFAULT key ("") to return saved data. + *

+ * This will check to see if there is an associated key for that data, if not - it will use data as the default + * + * @param data The data that will hold the copy of the data from disk + */ + @Override + public + T load(T data) throws IOException { + return load(this.defaultKey, data); + } + + /** + * Returns the saved data for the specified key. + * + * @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db) + */ + @Override + public + T load(String key, T data) throws IOException { + ByteArrayWrapper wrap = ByteArrayWrapper.wrap(key); + + return load(wrap, data); + } + + /** + * Returns the saved data for the specified key. + * + * @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db) + */ + @Override + public + T load(byte[] key, T data) throws IOException { + return load(ByteArrayWrapper.wrap(key), data); + } + + @SuppressWarnings("unchecked") + @Override + public + T load(final ByteArrayWrapper key, final T data) throws IOException { + final Object o = storage.get(key); + if (o == null) { + storage.put(key, data); + } + return (T) o; + } + + /** + * Saves the given data to storage with the associated key. + *

+ * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by + * deleting the old data and adding the new. + */ + @Override + public + void put(final String key, final Object data) { + put(ByteArrayWrapper.wrap(key), data); + } + + /** + * Saves the given data to storage with the associated key. + *

+ * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by + * deleting the old data and adding the new. + */ + @Override + public + void put(final byte[] key, final Object data) { + put(ByteArrayWrapper.wrap(key), data); + } + + /** + * Saves the given data to storage with the associated key. + *

+ * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by + * deleting the old data and adding the new. + */ + @Override + public + void put(final ByteArrayWrapper key, final Object object) { + storage.put(key, object); + } + + /** + * Saves the given data to storage with the associated key. + *

+ * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by + * deleting the old data and adding the new. + */ + @Override + public + void put(final Object data) { + put(defaultKey, data); + } + + /** + * Deletes an object from storage. To ALSO remove from the cache, use unRegister(key) + * + * @return true if the delete was successful. False if there were problems deleting the data. + */ + @Override + public + boolean delete(final String key) { + storage.remove(ByteArrayWrapper.wrap(key)); + return true; + } + + @Override + public + File getFile() { + return null; + } + + @Override + public + long getFileSize() { + return 0; + } + + @Override + public + boolean hasWriteWaiting() { + return false; + } + + @Override + public + long getSaveDelay() { + return 0; + } + + @Override + public + void setSaveDelay(final long milliSeconds) { + } + + @Override + public synchronized + int getVersion() { + return version; + } + + @Override + public synchronized + void setVersion(final int version) { + this.version = version; + } + + @Override + public + void commit() { + // no-op + } + + @Override + public + void commit(final String key, final Object object) { + // no-op + } + + @Override + public + void commit(final byte[] key, final Object object) { + // no-op + } + + @Override + public + void commit(final ByteArrayWrapper key, final Object object) { + // no-op + } +} diff --git a/Dorkbox-Util/src/dorkbox/util/storage/Metadata.java b/Dorkbox-Util/src/dorkbox/util/storage/Metadata.java index 8bae1cf..7bb8db7 100644 --- a/Dorkbox-Util/src/dorkbox/util/storage/Metadata.java +++ b/Dorkbox-Util/src/dorkbox/util/storage/Metadata.java @@ -15,6 +15,11 @@ */ package dorkbox.util.storage; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import dorkbox.util.SerializationManager; +import dorkbox.util.bytes.ByteArrayWrapper; + import java.io.ByteArrayOutputStream; import java.io.FileOutputStream; import java.io.IOException; @@ -24,22 +29,18 @@ import java.nio.channels.FileLock; import java.util.zip.DeflaterOutputStream; import java.util.zip.InflaterInputStream; -import com.esotericsoftware.kryo.Kryo; -import com.esotericsoftware.kryo.io.Input; -import com.esotericsoftware.kryo.io.Output; - -import dorkbox.util.bytes.ByteArrayWrapper; - -public class Metadata { +@SuppressWarnings("unused") +public +class Metadata { // The length of a key in the index. // SHA256 is 32 bytes long. - private static final int KEY_SIZE = 32; + private static final int KEY_SIZE = 32; // Number of bytes in the record header. private static final int POINTER_INFO_SIZE = 16; // The total length of one index entry - the key length plus the record header length. - static final int INDEX_ENTRY_LENGTH = KEY_SIZE + POINTER_INFO_SIZE; + static final int INDEX_ENTRY_LENGTH = KEY_SIZE + POINTER_INFO_SIZE; /** @@ -60,12 +61,12 @@ public class Metadata { /** * Actual number of bytes of data held in this record (4 bytes). */ - volatile int dataCount; + volatile int dataCount; /** * Number of bytes of data that this record can hold (4 bytes). */ - volatile int dataCapacity; + volatile int dataCapacity; /** @@ -78,7 +79,8 @@ public class Metadata { /** * Returns a file pointer in the index pointing to the first byte in the KEY located at the given index position. */ - static final long getMetaDataPointer(int position) { + static + long getMetaDataPointer(int position) { return StorageBase.FILE_HEADERS_REGION_LENGTH + INDEX_ENTRY_LENGTH * position; } @@ -86,17 +88,20 @@ public class Metadata { * Returns a file pointer in the index pointing to the first byte in the RECORD pointer located at the given index * position. */ - static final long getDataPointer(int position) { - long l = Metadata.getMetaDataPointer(position) + KEY_SIZE; - return l; + static + long getDataPointer(int position) { + return Metadata.getMetaDataPointer(position) + KEY_SIZE; } - private Metadata(ByteArrayWrapper key) { + private + Metadata(ByteArrayWrapper key) { this.key = key; } - /** we don't know how much data there is until AFTER we write the data */ + /** + * we don't know how much data there is until AFTER we write the data + */ Metadata(ByteArrayWrapper key, int recordIndex, long dataPointer) { this(key, recordIndex, dataPointer, 0); } @@ -123,11 +128,13 @@ public class Metadata { /** * Reads the ith HEADER (key + metadata) from the index. */ - static Metadata readHeader(RandomAccessFile file, int position) throws IOException { + static + Metadata readHeader(RandomAccessFile file, int position) throws IOException { byte[] buf = new byte[KEY_SIZE]; long origHeaderKeyPointer = Metadata.getMetaDataPointer(position); - FileLock lock = file.getChannel().lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true); + FileLock lock = file.getChannel() + .lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true); file.seek(origHeaderKeyPointer); file.readFully(buf); lock.release(); @@ -136,7 +143,8 @@ public class Metadata { r.indexPosition = position; long recordHeaderPointer = Metadata.getDataPointer(position); - lock = file.getChannel().lock(origHeaderKeyPointer, KEY_SIZE, true); + lock = file.getChannel() + .lock(origHeaderKeyPointer, KEY_SIZE, true); file.seek(recordHeaderPointer); r.dataPointer = file.readLong(); r.dataCapacity = file.readInt(); @@ -153,7 +161,8 @@ public class Metadata { void writeMetaDataInfo(RandomAccessFile file) throws IOException { long recordKeyPointer = Metadata.getMetaDataPointer(this.indexPosition); - FileLock lock = file.getChannel().lock(recordKeyPointer, KEY_SIZE, false); + FileLock lock = file.getChannel() + .lock(recordKeyPointer, KEY_SIZE, false); file.seek(recordKeyPointer); file.write(this.key.getBytes()); lock.release(); @@ -162,7 +171,8 @@ public class Metadata { void writeDataInfo(RandomAccessFile file) throws IOException { long recordHeaderPointer = getDataPointer(this.indexPosition); - FileLock lock = file.getChannel().lock(recordHeaderPointer, POINTER_INFO_SIZE, false); + FileLock lock = file.getChannel() + .lock(recordHeaderPointer, POINTER_INFO_SIZE, false); file.seek(recordHeaderPointer); file.writeLong(this.dataPointer); file.writeInt(this.dataCapacity); @@ -177,13 +187,17 @@ public class Metadata { byte[] buf = new byte[KEY_SIZE]; long origHeaderKeyPointer = Metadata.getMetaDataPointer(this.indexPosition); - FileLock lock = file.getChannel().lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true); + FileLock lock = file.getChannel() + .lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true); + file.seek(origHeaderKeyPointer); file.readFully(buf); lock.release(); long newHeaderKeyPointer = Metadata.getMetaDataPointer(newIndex); - lock = file.getChannel().lock(newHeaderKeyPointer, INDEX_ENTRY_LENGTH, false); + lock = file.getChannel() + .lock(newHeaderKeyPointer, INDEX_ENTRY_LENGTH, false); + file.seek(newHeaderKeyPointer); file.write(buf); lock.release(); @@ -205,7 +219,8 @@ public class Metadata { this.dataPointer = position; this.dataCapacity = this.dataCount; - FileLock lock = file.getChannel().lock(position, this.dataCount, false); + FileLock lock = file.getChannel() + .lock(position, this.dataCount, false); // update the file size file.setLength(position + this.dataCount); @@ -230,7 +245,8 @@ public class Metadata { byte[] buf = new byte[this.dataCount]; // System.err.print("Reading data: " + this.indexPosition + " @ " + this.dataPointer + "-" + (this.dataPointer+this.dataCount) + " -- "); - FileLock lock = file.getChannel().lock(this.dataPointer, this.dataCount, true); + FileLock lock = file.getChannel() + .lock(this.dataPointer, this.dataCount, true); file.seek(this.dataPointer); file.readFully(buf); lock.release(); @@ -239,24 +255,29 @@ public class Metadata { return buf; } - /** - * Reads the record data for the given record header. - */ + /** + * Reads the record data for the given record header. + */ - T readData(Kryo kryo, InflaterInputStream inputStream) { + static + T readData(SerializationManager serializationManager, InflaterInputStream inputStream) { Input input = new Input(inputStream, 1024); // read 1024 at a time @SuppressWarnings("unchecked") - T readObject = (T) kryo.readClassAndObject(input); + T readObject = (T) serializationManager.readClassAndObject(input); return readObject; } /** * Writes data to the end of the file (which is where the datapointer is at) */ - void writeDataFast(Kryo kryo, Object data, DeflaterOutputStream outputStream) throws IOException { + static + void writeDataFast(final SerializationManager serializationManager, + final Object data, + final RandomAccessFile file, + final DeflaterOutputStream outputStream) throws IOException { // HAVE TO LOCK BEFORE THIS IS CALLED! (AND FREE AFTERWARDS!) Output output = new Output(outputStream, 1024); // write 1024 at a time - kryo.writeClassAndObject(output, data); + serializationManager.writeClassAndObject(output, data); output.flush(); outputStream.flush(); // sync-flush is enabled, so the output stream will finish compressing data. @@ -265,7 +286,8 @@ public class Metadata { void writeData(ByteArrayOutputStream byteArrayOutputStream, RandomAccessFile file) throws IOException { this.dataCount = byteArrayOutputStream.size(); - FileLock lock = file.getChannel().lock(this.dataPointer, this.dataCount, false); + FileLock lock = file.getChannel() + .lock(this.dataPointer, this.dataCount, false); FileOutputStream out = new FileOutputStream(file.getFD()); file.seek(this.dataPointer); @@ -275,8 +297,9 @@ public class Metadata { } @Override - public String toString() { - return "RecordHeader [dataPointer=" + this.dataPointer + ", dataCount=" + this.dataCount + ", dataCapacity=" - + this.dataCapacity + ", indexPosition=" + this.indexPosition + "]"; + public + String toString() { + return "RecordHeader [dataPointer=" + this.dataPointer + ", dataCount=" + this.dataCount + ", dataCapacity=" + this.dataCapacity + + ", indexPosition=" + this.indexPosition + "]"; } } diff --git a/Dorkbox-Util/src/dorkbox/util/storage/Storage.java b/Dorkbox-Util/src/dorkbox/util/storage/Storage.java deleted file mode 100644 index c0c9713..0000000 --- a/Dorkbox-Util/src/dorkbox/util/storage/Storage.java +++ /dev/null @@ -1,568 +0,0 @@ -/* - * Copyright 2014 dorkbox, llc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package dorkbox.util.storage; - -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; - -import org.bouncycastle.crypto.digests.SHA256Digest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import dorkbox.util.DelayTimer; -import dorkbox.util.OS; -import dorkbox.util.bytes.ByteArrayWrapper; - -/** - * Nothing spectacular about this storage -- it allows for persistent storage of objects to disk. - */ -public class Storage { - private static final Logger logger = LoggerFactory.getLogger(Storage.class); - - private static Map storages = new HashMap(1); - - public static Storage open(String file) { - if (file == null || file.isEmpty()) { - throw new IllegalArgumentException("file cannot be null or empty!"); - } - return open(new File(file)); - } - - /** - * Two types of storage. - * Raw) save/load a single object to disk (better for really large files) - * Normal) save/load key/value objects to disk (better for multiple types of data in a single file) - */ - public static Storage open(File file) { - if (file == null) { - throw new IllegalArgumentException("file cannot be null!"); - } - - // if we load from a NEW storage at the same location as an ALREADY EXISTING storage, - // without saving the existing storage first --- whoops! - synchronized (storages) { - Storage storage = storages.get(file); - - if (storage != null) { - boolean waiting = storage.hasWriteWaiting(); - // we want this storage to be in a fresh state - if (waiting) { - storage.saveNow(); - } - storage.increaseReference(); - } else { - try { - storage = new Storage(file); - storages.put(file, storage); - } catch (IOException e) { - logger.error("Unable to open storage", e); - } - } - - return storage; - } - } - - - /** - * Closes the storage. - */ - public static void close(File file) { - synchronized (storages) { - Storage storage = storages.get(file); - if (storage != null) { - boolean isLastOne = storage.decrementReference(); - if (isLastOne) { - storage.close(); - storages.remove(file); - } - } - } - } - - /** - * Closes the storage. - */ - public static void close(Storage _storage) { - synchronized (storages) { - File file = _storage.getFile(); - Storage storage = storages.get(file); - if (storage != null) { - boolean isLastOne = storage.decrementReference(); - if (isLastOne) { - storage.close(); - storages.remove(file); - } - } - } - } - - public static void shutdown() { - synchronized(storages) { - Collection values = storages.values(); - for (Storage storage : values) { - while (!storage.decrementReference()) { - } - storage.close(); - } - storages.clear(); - } - } - - public static void delete(File file) { - synchronized(storages) { - Storage remove = storages.remove(file); - if (remove != null) { - remove.close(); - } - file.delete(); - } - } - - public static void delete(Storage storage) { - File file = storage.getFile(); - delete(file); - } - - - private static void copyFields(Object source, Object dest) { - Class sourceClass = source.getClass(); - Class destClass = dest.getClass(); - - if (sourceClass != destClass) { - throw new IllegalArgumentException("Source and Dest objects are not of the same class!"); - } - - // have to walk up the object hierarchy. - while (destClass != Object.class) { - Field[] destFields = destClass.getDeclaredFields(); - - for (Field destField : destFields) { - String name = destField.getName(); - try { - Field sourceField = sourceClass.getDeclaredField(name); - destField.setAccessible(true); - sourceField.setAccessible(true); - - Object sourceObj = sourceField.get(source); - - if (sourceObj instanceof Map) { - Object destObj = destField.get(dest); - if (destObj == null) { - destField.set(dest, sourceObj); - } else if (destObj instanceof Map) { - @SuppressWarnings("unchecked") - Map sourceMap = (Map) sourceObj; - @SuppressWarnings("unchecked") - Map destMap = (Map) destObj; - - destMap.clear(); - Iterator entries = sourceMap.entrySet().iterator(); - while (entries.hasNext()) { - Map.Entry entry = (Map.Entry)entries.next(); - Object key = entry.getKey(); - Object value = entry.getValue(); - destMap.put(key, value); - } - - } else { - logger.error("Incompatible field type! '{}'", name); - } - } else { - destField.set(dest, sourceObj); - } - } catch (Exception e) { - logger.error("Unable to copy field: {}", name, e); - } - } - - destClass = destClass.getSuperclass(); - sourceClass = sourceClass.getSuperclass(); - } - } - - - - - - - private volatile long milliSeconds = 3000L; - private volatile DelayTimer timer; - - private final ByteArrayWrapper defaultKey; - private final StorageBase storage; - - private AtomicInteger references = new AtomicInteger(1); - - private final ReentrantLock actionLock = new ReentrantLock(); - private volatile Map actionMap = new ConcurrentHashMap(); - - private AtomicBoolean isOpen = new AtomicBoolean(false); - - - - /** - * Creates or opens a new database file. - */ - private Storage(File storageFile) throws IOException { - this.storage = new StorageBase(storageFile); - this.defaultKey = wrap(""); - - this.timer = new DelayTimer("Storage Writer", false, new DelayTimer.Callback() { - @Override - public void execute() { - Map actions = Storage.this.actionMap; - - ReentrantLock actionLock2 = Storage.this.actionLock; - - try { - actionLock2.lock(); - - // do a fast swap on the actionMap. - Storage.this.actionMap = new ConcurrentHashMap(); - } finally { - actionLock2.unlock(); - } - - Storage.this.storage.doActionThings(actions); - } - }); - - this.isOpen.set(true); - } - - /** - * Returns the number of objects in the database. - *

- * SLOW because this must save all data to disk first! - */ - public final int size() { - if (!this.isOpen.get()) { - throw new RuntimeException("Unable to act on closed storage"); - } - - // flush actions - // timer action runs on THIS thread, not timer thread - this.timer.delay(0L); - - return this.storage.size(); - } - - /** - * Checks if there is a object corresponding to the given key. - */ - public final boolean contains(String key) { - if (!this.isOpen.get()) { - throw new RuntimeException("Unable to act on closed storage"); - } - - return this.storage.contains(wrap(key)); - } - - /** - * Reads a object using the default (blank) key, and casts it to the expected class - */ - public final T get() { - return get0(this.defaultKey); - } - - /** - * Reads a object using the specific key, and casts it to the expected class - */ - public final T get(String key) { - ByteArrayWrapper wrap = wrap(key); - return get0(wrap); - } - - /** - * Copies the saved data (from) into the passed-in data. This just assigns all of the values from one to the other. - *

- * This will check to see if there is an associated key for that data, if not - it will use the default (blank) - * - * @param data The data that will hold the copy of the data from disk - */ - public void load(Object data) { - Object source = get(); - - if (source != null) { - Storage.copyFields(source, data); - } - } - - /** - * Copies the saved data (from) into the passed-in data. This just assigns all of the values from one to the other. - *

- * The key/data will be associated for the lifetime of the object. - * - * @param object The object that will hold the copy of the data from disk (but not change the reference) once this is completed - */ - public void load(String key, Object object) { - ByteArrayWrapper wrap = wrap(key); - - Object source = get0(wrap); - if (source != null) { - Storage.copyFields(source, object); - } - } - - /** - * Reads a object from pending or from storage - */ - private final T get0(ByteArrayWrapper key) { - if (!this.isOpen.get()) { - throw new RuntimeException("Unable to act on closed storage"); - } - - // if the object in is pending, we get it from there - try { - this.actionLock.lock(); - - Object object = this.actionMap.get(key); - - if (object != null) { - @SuppressWarnings("unchecked") - T returnObject = (T) object; - return returnObject; - } - } finally { - this.actionLock.unlock(); - } - - // not found, so we have to go find it on disk - return this.storage.get(key); - } - - /** - * Save the storage to disk, once xxxx milli-seconds have passed. - * This is to help prevent thrashing the disk, or wearing it out on multiple, rapid, changes. - *

- * This will save the ALL of the pending save actions to the file - */ - public final void saveNow() { - if (!this.isOpen.get()) { - throw new RuntimeException("Unable to act on closed storage"); - } - - // timer action runs on THIS thread, not timer thread - this.timer.delay(0L); - } - - /** - * Saves the given data to storage with the associated key. - *

- * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by - * deleting the old data and adding the new. - */ - public final void save(String key, Object object) { - if (!this.isOpen.get()) { - throw new RuntimeException("Unable to act on closed storage"); - } - - ByteArrayWrapper wrap = wrap(key); - action(wrap, object); - - // timer action runs on TIMER thread, not this thread - this.timer.delay(this.milliSeconds); - } - - /** - * Saves the given object to storage with the associated key. - *

- * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by - * deleting the old data and adding the new. - */ - public final void saveNow(String key, Object object) { - if (!this.isOpen.get()) { - throw new RuntimeException("Unable to act on closed storage"); - } - - ByteArrayWrapper wrap = wrap(key); - action(wrap, object); - - // timer action runs on THIS thread, not timer thread - this.timer.delay(0L); - } - - /** - * Adds the given object to the storage using a default (blank) key, OR -- if it has been registered, using it's registered key - *

- * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by - * deleting the old data and adding the new. - */ - public final void save(Object object) { - if (!this.isOpen.get()) { - throw new RuntimeException("Unable to act on closed storage"); - } - - action(this.defaultKey, object); - - // timer action runs on TIMER thread, not this thread - this.timer.delay(this.milliSeconds); - } - - /** - * Adds the given object to the storage using a default (blank) key - *

- * Also will update existing data. If the new contents do not fit in the original space, then the update is handled by - * deleting the old data and adding the new. - */ - public final void saveNow(Object object) { - if (!this.isOpen.get()) { - throw new RuntimeException("Unable to act on closed storage"); - } - - action(this.defaultKey, object); - - // timer action runs on THIS thread, not timer thread - this.timer.delay(0L); - } - - /** - * Deletes an object from storage. To ALSO remove from the cache, use unRegister(key) - * - * @return true if the delete was successful. False if there were problems deleting the data. - */ - public final boolean delete(String key) { - if (!this.isOpen.get()) { - throw new RuntimeException("Unable to act on closed storage"); - } - - ByteArrayWrapper wrap = wrap(key); - - // timer action runs on THIS thread, not timer thread - this.timer.delay(0L); - - return this.storage.delete(wrap); - } - - /** - * Closes the database and file. - */ - private final void close() { - // timer action runs on THIS thread, not timer thread - this.timer.delay(0L); - - // have to "close" it after we run the timer! - this.isOpen.set(false); - - this.storage.close(); - } - - /** - * @return the file that backs this storage - */ - public final File getFile() { - return this.storage.getFile(); - } - - /** - * Gets the backing file size. - * - * @return -1 if there was an error - */ - public final long getFileSize() { - // timer action runs on THIS thread, not timer thread - this.timer.delay(0L); - - return this.storage.getFileSize(); - } - - - /** - * @return true if there are objects queued to be written? - */ - public final boolean hasWriteWaiting() { - if (!this.isOpen.get()) { - throw new RuntimeException("Unable to act on closed storage"); - } - - return this.timer.isWaiting(); - } - - /** - * @param delay milliseconds to wait - */ - public final void setSaveDelay(long milliSeconds) { - if (!this.isOpen.get()) { - throw new RuntimeException("Unable to act on closed storage"); - } - - this.milliSeconds = milliSeconds; - } - - /** - * @return the delay in milliseconds this will wait after the last action to flush the data to the disk - */ - public final long getSaveDelay() { - return this.milliSeconds; - } - - /** - * @return the version of data stored in the database - */ - public final int getVersion() { - return this.storage.getVersion(); - } - - /** - * Sets the version of data stored in the database - */ - public final void setVersion(int version) { - this.storage.setVersion(version); - } - - - private final ByteArrayWrapper wrap(String key) { - byte[] bytes = key.getBytes(OS.UTF_8); - - SHA256Digest digest = new SHA256Digest(); - digest.update(bytes, 0, bytes.length); - byte[] hashBytes = new byte[digest.getDigestSize()]; - digest.doFinal(hashBytes, 0); - ByteArrayWrapper wrap = ByteArrayWrapper.wrap(hashBytes); - - return wrap; - } - - private void action(ByteArrayWrapper key, Object object) { - try { - this.actionLock.lock(); - - // push action to map - this.actionMap.put(key, object); - } finally { - this.actionLock.unlock(); - } - } - - private final void increaseReference() { - this.references.incrementAndGet(); - } - - /** return true when this is the last reference */ - private final boolean decrementReference() { - return this.references.decrementAndGet() == 0; - } -} diff --git a/Dorkbox-Util/src/dorkbox/util/storage/StorageBase.java b/Dorkbox-Util/src/dorkbox/util/storage/StorageBase.java index 487120b..0d647fe 100644 --- a/Dorkbox-Util/src/dorkbox/util/storage/StorageBase.java +++ b/Dorkbox-Util/src/dorkbox/util/storage/StorageBase.java @@ -15,16 +15,16 @@ */ package dorkbox.util.storage; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.io.RandomAccessFile; +import com.esotericsoftware.kryo.KryoException; +import com.esotericsoftware.kryo.io.Output; +import dorkbox.util.SerializationManager; +import dorkbox.util.bytes.ByteArrayWrapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; import java.lang.ref.WeakReference; import java.nio.channels.FileLock; -import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; @@ -35,45 +35,39 @@ import java.util.zip.DeflaterOutputStream; import java.util.zip.Inflater; import java.util.zip.InflaterInputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.esotericsoftware.kryo.Kryo; -import com.esotericsoftware.kryo.KryoException; -import com.esotericsoftware.kryo.io.Output; - -import dorkbox.util.bytes.ByteArrayWrapper; // a note on file locking between c and java // http://panks-dev.blogspot.de/2008/04/linux-file-locks-java-and-others.html // Also, file locks on linux are ADVISORY. if an app doesn't care about locks, then it can do stuff -- even if locked by another app -public class StorageBase { + +@SuppressWarnings("unused") +public +class StorageBase { private final Logger logger = LoggerFactory.getLogger(getClass()); // File pointer to the data start pointer header. - private static final long VERSION_HEADER_LOCATION = 0; + private static final long VERSION_HEADER_LOCATION = 0; // File pointer to the num records header. private static final long NUM_RECORDS_HEADER_LOCATION = 4; // File pointer to the data start pointer header. - private static final long DATA_START_HEADER_LOCATION = 8; + private static final long DATA_START_HEADER_LOCATION = 8; // Total length in bytes of the global database headers. - static final int FILE_HEADERS_REGION_LENGTH = 16; + static final int FILE_HEADERS_REGION_LENGTH = 16; // The in-memory index (for efficiency, all of the record info is cached in memory). private final Map memoryIndex; // determines how much the index will grow by - private Float weight; + private final Float weight; // The keys are weak! When they go, the map entry is removed! private final ReentrantLock referenceLock = new ReentrantLock(); - private volatile Map referencePendingWrite = new HashMap(); private final File baseFile; @@ -97,32 +91,29 @@ public class StorageBase { // save references to these, so they don't have to be created/destroyed any time there is I/O - private final Kryo kryo; + private final SerializationManager serializationManager; - private Deflater deflater; - private DeflaterOutputStream outputStream; + private final Deflater deflater; + private final DeflaterOutputStream outputStream; - private Inflater inflater; - private InflaterInputStream inputStream; + private final Inflater inflater; + private final InflaterInputStream inputStream; /** * Creates or opens a new database file. */ - StorageBase(String filePath) throws IOException { - this(new File(filePath)); - } + StorageBase(File filePath, SerializationManager serializationManager) throws IOException { + this.serializationManager = serializationManager; - /** - * Creates or opens a new database file. - */ - StorageBase(File filePath) throws IOException { this.logger.info("Opening storage file: '{}'", filePath.getAbsolutePath()); this.baseFile = filePath; File parentFile = this.baseFile.getParentFile(); - if (parentFile != null) { - parentFile.mkdirs(); + if (parentFile != null && !parentFile.exists()) { + if (!parentFile.mkdirs()) { + throw new IOException("Unable to create dirs for: " + filePath); + } } this.file = new RandomAccessFile(this.baseFile, "rw"); @@ -132,7 +123,8 @@ public class StorageBase { this.databaseVersion = this.file.readInt(); this.numberOfRecords = this.file.readInt(); this.dataPosition = this.file.readLong(); - } else { + } + else { setVersionNumber(this.file, 0); // always start off with 4 records @@ -149,18 +141,16 @@ public class StorageBase { throw new IllegalArgumentException("Unable to parse header information from storage. Maybe it's corrupted?"); } + //noinspection AutoBoxing this.logger.info("Storage version: {}", this.databaseVersion); - - this.kryo = new Kryo(); - this.kryo.setRegistrationRequired(false); - this.deflater = new Deflater(7, true); this.outputStream = new DeflaterOutputStream(new FileOutputStream(this.file.getFD()), this.deflater, 65536, true); this.inflater = new Inflater(true); this.inputStream = new InflaterInputStream(new FileInputStream(this.file.getFD()), this.inflater, 65536); + //noinspection AutoBoxing this.weight = .5F; this.memoryIndex = new ConcurrentHashMap(this.numberOfRecords); @@ -183,7 +173,8 @@ public class StorageBase { /** * Returns the current number of records in the database. */ - final int size() { + final + int size() { // wrapper flushes first (protected by lock) // not protected by lock return this.memoryIndex.size(); @@ -192,21 +183,19 @@ public class StorageBase { /** * Checks if there is a record belonging to the given key. */ - final boolean contains(ByteArrayWrapper key) { + final + boolean contains(ByteArrayWrapper key) { // protected by lock // check to see if it's in the pending ops - if (this.referencePendingWrite.containsKey(key)) { - return true; - } - return this.memoryIndex.containsKey(key); } /** * @return an object for a specified key ONLY FROM THE REFERENCE CACHE */ - final T getCached(ByteArrayWrapper key) { + final + T getCached(ByteArrayWrapper key) { // protected by lock Metadata meta = this.memoryIndex.get(key); @@ -223,7 +212,7 @@ public class StorageBase { if (ref != null) { @SuppressWarnings("unchecked") - T referenceObject = (T) ref.get(); + T referenceObject = (T) ref.get(); return referenceObject; } } finally { @@ -236,7 +225,8 @@ public class StorageBase { /** * @return an object for a specified key form referenceCache FIRST, then from DISK */ - final T get(ByteArrayWrapper key) { + final + T get(ByteArrayWrapper key) { // NOT protected by lock Metadata meta = this.memoryIndex.get(key); @@ -253,7 +243,7 @@ public class StorageBase { if (ref != null) { @SuppressWarnings("unchecked") - T referenceObject = (T) ref.get(); + T referenceObject = (T) ref.get(); return referenceObject; } } finally { @@ -266,7 +256,7 @@ public class StorageBase { this.inflater.reset(); this.file.seek(meta.dataPointer); - T readRecordData = meta.readData(this.kryo, this.inputStream); + T readRecordData = Metadata.readData(this.serializationManager, this.inputStream); if (readRecordData != null) { // now stuff it into our reference cache for future lookups! @@ -281,10 +271,10 @@ public class StorageBase { return readRecordData; } catch (KryoException e) { - this.logger.error("Error while geting data from disk. Ignoring previous value."); + this.logger.error("Error while getting data from disk. Ignoring previous value."); return null; } catch (Exception e) { - this.logger.error("Error while geting data from disk", e); + this.logger.error("Error while getting data from disk", e); return null; } } @@ -294,7 +284,8 @@ public class StorageBase { * * @return true if the delete was successful. False if there were problems deleting the data. */ - final boolean delete(ByteArrayWrapper key) { + final + boolean delete(ByteArrayWrapper key) { // pending ops flushed (protected by lock) // not protected by lock Metadata delRec = this.memoryIndex.get(key); @@ -312,11 +303,13 @@ public class StorageBase { /** * Closes the database and file. */ - final void close() { + final + void close() { // pending ops flushed (protected by lock) // not protected by lock try { - this.file.getFD().sync(); + this.file.getFD() + .sync(); this.file.close(); this.memoryIndex.clear(); @@ -344,22 +337,22 @@ public class StorageBase { /** * @return the file that backs this storage */ - final File getFile() { + final + File getFile() { return this.baseFile; } /** * Saves the given data to storage. - *

+ *

* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by * deleting the old data and adding the new. - *

+ *

* Will also save the object in a cache. - * - * @return the metadata for the saved object */ - private final void save0(ByteArrayWrapper key, Object object, DeflaterOutputStream fileOutputStream, Deflater deflater) { + private + void save0(ByteArrayWrapper key, Object object, DeflaterOutputStream fileOutputStream, Deflater deflater) { deflater.reset(); Metadata metaData = this.memoryIndex.get(key); @@ -370,19 +363,23 @@ public class StorageBase { try { if (currentRecordCount == 1) { // if we are the ONLY one, then we can do things differently. - // just dump the data agian to disk. - FileLock lock = this.file.getChannel().lock(this.dataPosition, Long.MAX_VALUE-this.dataPosition, false); // don't know how big it is, so max value it + // just dump the data again to disk. + FileLock lock = this.file.getChannel() + .lock(this.dataPosition, + Long.MAX_VALUE - this.dataPosition, + false); // don't know how big it is, so max value it this.file.seek(this.dataPosition); // this is the end of the file, we know this ahead-of-time - metaData.writeDataFast(this.kryo, object, fileOutputStream); + Metadata.writeDataFast(this.serializationManager, object, file, fileOutputStream); // have to re-specify the capacity and size int sizeOfWrittenData = (int) (this.file.length() - this.dataPosition); metaData.dataCapacity = sizeOfWrittenData; metaData.dataCount = sizeOfWrittenData; lock.release(); - } else { + } + else { // this is comparatively slow, since we serialize it first to get the size, then we put it in the file. - ByteArrayOutputStream dataStream = getDataAsByteArray(this.kryo, object, deflater); + ByteArrayOutputStream dataStream = getDataAsByteArray(this.serializationManager, object, deflater); int size = dataStream.size(); if (size > metaData.dataCapacity) { @@ -404,10 +401,11 @@ public class StorageBase { } catch (IOException e) { this.logger.error("Error while saving data to disk", e); } - } else { + } + else { try { // try to move the read head in order - setRecordCount(this.file, currentRecordCount+1); + setRecordCount(this.file, currentRecordCount + 1); // This will make sure that there is room to write a new record. This is zero indexed. // this will skip around if moves occur @@ -425,9 +423,10 @@ public class StorageBase { // save out the data. Because we KNOW that we are writing this to the end of the file, // there are some tricks we can use. - FileLock lock = this.file.getChannel().lock(length, Long.MAX_VALUE-length, false); // don't know how big it is, so max value it + FileLock lock = this.file.getChannel() + .lock(length, Long.MAX_VALUE - length, false); // don't know how big it is, so max value it this.file.seek(length); // this is the end of the file, we know this ahead-of-time - metaData.writeDataFast(this.kryo, object, fileOutputStream); + Metadata.writeDataFast(this.serializationManager, object, file, fileOutputStream); lock.release(); metaData.dataCount = deflater.getTotalOut(); @@ -445,7 +444,8 @@ public class StorageBase { } - private ByteArrayOutputStream getDataAsByteArray(Kryo kryo, Object data, Deflater deflater) throws IOException { + private static + ByteArrayOutputStream getDataAsByteArray(SerializationManager kryo, Object data, Deflater deflater) throws IOException { ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); OutputStream outputStream = new DeflaterOutputStream(byteArrayOutputStream, deflater); Output output = new Output(outputStream, 1024); // write 1024 at a time @@ -484,23 +484,27 @@ public class StorageBase { /** * "intelligent" move strategy. - * + *

* we should increase by some weight (ie: .5) would increase the number of allocated * record headers by 50%, instead of just incrementing them by one at a time. */ - private final int getWeightedNewRecordCount(int numberOfRecords) { - int newNumberOfRecords = numberOfRecords + 1 + (int) (numberOfRecords * this.weight); // int used for rounding - return newNumberOfRecords; + private + int getWeightedNewRecordCount(int numberOfRecords) { + //noinspection AutoUnboxing + return numberOfRecords + 1 + (int) (numberOfRecords * this.weight); } - private void deleteRecordData(Metadata deletedRecord, int sizeOfDataToAdd) throws IOException { + private + void deleteRecordData(Metadata deletedRecord, int sizeOfDataToAdd) throws IOException { if (this.file.length() == deletedRecord.dataPointer + deletedRecord.dataCapacity) { // shrink file since this is the last record in the file - FileLock lock = this.file.getChannel().lock(deletedRecord.dataPointer, Long.MAX_VALUE-deletedRecord.dataPointer, false); + FileLock lock = this.file.getChannel() + .lock(deletedRecord.dataPointer, Long.MAX_VALUE - deletedRecord.dataPointer, false); this.file.setLength(deletedRecord.dataPointer); lock.release(); - } else { + } + else { // we MIGHT be the FIRST record Metadata first = index_getMetaDataFromData(this.dataPosition); if (first == deletedRecord) { @@ -515,22 +519,25 @@ public class StorageBase { long endIndexPointer = Metadata.getMetaDataPointer(newNumberOfRecords); long endOfDataPointer = deletedRecord.dataPointer + deletedRecord.dataCapacity; - long newEndOfDataPointer = endOfDataPointer-sizeOfDataToAdd; + long newEndOfDataPointer = endOfDataPointer - sizeOfDataToAdd; if (endIndexPointer < this.dataPosition && endIndexPointer <= newEndOfDataPointer) { // one option is to shrink the RECORD section to fit the new data setDataPosition(this.file, newEndOfDataPointer); - } else { + } + else { // option two is to grow the RECORD section, and put the data at the end of the file setDataPosition(this.file, endOfDataPointer); } - } else { + } + else { Metadata previous = index_getMetaDataFromData(deletedRecord.dataPointer - 1); if (previous != null) { // append space of deleted record onto previous record previous.dataCapacity += deletedRecord.dataCapacity; previous.writeDataInfo(this.file); - } else { + } + else { // because there is no "previous", that means we MIGHT be the FIRST record // well, we're not the first record. which one is RIGHT before us? // it should be "previous", so something fucked up @@ -540,11 +547,14 @@ public class StorageBase { } } - private void deleteRecordIndex(ByteArrayWrapper key, Metadata deleteRecord) throws IOException { + private + void deleteRecordIndex(ByteArrayWrapper key, Metadata deleteRecord) throws IOException { int currentNumRecords = this.memoryIndex.size(); if (deleteRecord.indexPosition != currentNumRecords - 1) { Metadata last = Metadata.readHeader(this.file, currentNumRecords - 1); + assert last != null; + last.move(this.file, deleteRecord.indexPosition); } @@ -557,10 +567,12 @@ public class StorageBase { /** * Writes the number of records header to the file. */ - private final void setVersionNumber(RandomAccessFile file, int versionNumber) throws IOException { + private + void setVersionNumber(RandomAccessFile file, int versionNumber) throws IOException { this.databaseVersion = versionNumber; - FileLock lock = this.file.getChannel().lock(VERSION_HEADER_LOCATION, 4, false); + FileLock lock = this.file.getChannel() + .lock(VERSION_HEADER_LOCATION, 4, false); file.seek(VERSION_HEADER_LOCATION); file.writeInt(versionNumber); lock.release(); @@ -569,10 +581,12 @@ public class StorageBase { /** * Writes the number of records header to the file. */ - private final void setRecordCount(RandomAccessFile file, int numberOfRecords) throws IOException { + private + void setRecordCount(RandomAccessFile file, int numberOfRecords) throws IOException { this.numberOfRecords = numberOfRecords; - FileLock lock = this.file.getChannel().lock(NUM_RECORDS_HEADER_LOCATION, 4, false); + FileLock lock = this.file.getChannel() + .lock(NUM_RECORDS_HEADER_LOCATION, 4, false); file.seek(NUM_RECORDS_HEADER_LOCATION); file.writeInt(numberOfRecords); lock.release(); @@ -581,10 +595,12 @@ public class StorageBase { /** * Writes the data start position to the file. */ - private final void setDataPosition(RandomAccessFile file, long dataPositionPointer) throws IOException { + private + void setDataPosition(RandomAccessFile file, long dataPositionPointer) throws IOException { this.dataPosition = dataPositionPointer; - FileLock lock = this.file.getChannel().lock(DATA_START_HEADER_LOCATION, 8, false); + FileLock lock = this.file.getChannel() + .lock(DATA_START_HEADER_LOCATION, 8, false); file.seek(DATA_START_HEADER_LOCATION); file.writeLong(dataPositionPointer); lock.release(); @@ -608,9 +624,12 @@ public class StorageBase { * of the record data of the RecordHeader which is returned. Returns null if the location is not part of a record. * (O(n) mem accesses) */ - private final Metadata index_getMetaDataFromData(long targetFp) { - Iterator iterator = this.memoryIndex.values().iterator(); + private + Metadata index_getMetaDataFromData(long targetFp) { + Iterator iterator = this.memoryIndex.values() + .iterator(); + //noinspection WhileLoopReplaceableByForEach while (iterator.hasNext()) { Metadata next = iterator.next(); if (targetFp >= next.dataPointer && targetFp < next.dataPointer + next.dataCapacity) { @@ -623,11 +642,13 @@ public class StorageBase { /** - * Ensure index capacity. This operation makes sure the INDEX REGION is large enough to accommodate additional entries. + * Ensure index capacity. This operation makes sure the INDEX REGION is large enough to accommodate additional entries. */ - private final void ensureIndexCapacity(RandomAccessFile file) throws IOException { + private + void ensureIndexCapacity(RandomAccessFile file) throws IOException { int numberOfRecords = this.numberOfRecords; // because we are zero indexed, this is ALSO the index where the record will START - long endIndexPointer = Metadata.getMetaDataPointer(numberOfRecords + 1); // +1 because this is where that index will END (the start of the NEXT one) + long endIndexPointer = Metadata.getMetaDataPointer( + numberOfRecords + 1); // +1 because this is where that index will END (the start of the NEXT one) // just set the data position to the end of the file, since we don't have any data yet. if (endIndexPointer > file.length() && numberOfRecords == 0) { diff --git a/Dorkbox-Util/test/dorkbox/util/StorageTest.java b/Dorkbox-Util/test/dorkbox/util/StorageTest.java index dd25b96..c160077 100644 --- a/Dorkbox-Util/test/dorkbox/util/StorageTest.java +++ b/Dorkbox-Util/test/dorkbox/util/StorageTest.java @@ -1,44 +1,118 @@ package dorkbox.util; -import static org.junit.Assert.fail; +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Registration; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; +import dorkbox.util.storage.DiskStorageIfface; +import dorkbox.util.storage.MakeStorage; +import io.netty.buffer.ByteBuf; +import org.junit.*; +import org.junit.runners.MethodSorters; import java.io.File; import java.io.IOException; -import java.io.OptionalDataException; import java.util.Arrays; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.FixMethodOrder; -import org.junit.Test; -import org.junit.runners.MethodSorters; - -import dorkbox.util.storage.Storage; - @FixMethodOrder(MethodSorters.NAME_ASCENDING) -public class StorageTest { +public +class StorageTest { - private static final String TEST_DB = "sampleFile.records"; + private static final File TEST_DB = new File("sampleFile.records"); + private static final SerializationManager manager = new SerializationManager() { + Kryo kryo = new Kryo(); - static void log(String s) { + @Override + public + boolean setReferences(final boolean references) { + kryo.setReferences(references); + return false; + } + + @Override + public + void setRegistrationRequired(final boolean registrationRequired) { + kryo.setRegistrationRequired(registrationRequired); + } + + @Override + public + void register(final Class clazz) { + kryo.register(clazz); + } + + @Override + public + void register(final Class clazz, final Serializer serializer) { + kryo.register(clazz, serializer); + } + + @Override + public + Registration register(final Class type, final Serializer serializer, final int id) { + kryo.register(type, serializer, id); + return null; + } + + @Override + public + void write(final ByteBuf buffer, final Object message) { + final Output output = new Output(); + writeClassAndObject(output, message); + buffer.writeBytes(output.getBuffer()); + } + + @Override + public + Object read(final ByteBuf buffer, final int length) { + final Input input = new Input(); + buffer.readBytes(input.getBuffer()); + + final Object o = readClassAndObject(input); + buffer.skipBytes(input.position()); + + return o; + } + + @Override + public + void writeClassAndObject(final Output output, final Object value) { + kryo.writeClassAndObject(output, value); + } + + @Override + public + Object readClassAndObject(final Input input) { + return kryo.readClassAndObject(input); + } + }; + + static + void log(String s) { System.err.println(s); } @Before - public void deleteDB() { - Storage.delete(new File(TEST_DB)); + public + void deleteDB() { + MakeStorage.delete(TEST_DB); } @After - public void delete2DB() { - Storage.delete(new File(TEST_DB)); + public + void delete2DB() { + MakeStorage.delete(TEST_DB); } @Test - public void testCreateDB() throws IOException { - Storage storage = Storage.open(TEST_DB); + public + void testCreateDB() throws IOException { + DiskStorageIfface storage = MakeStorage.Disk() + .file(TEST_DB) + .serializer(manager) + .make(); int numberOfRecords1 = storage.size(); long size1 = storage.getFileSize(); @@ -46,67 +120,84 @@ public class StorageTest { Assert.assertEquals("count is not correct", numberOfRecords1, 0); Assert.assertEquals("size is not correct", size1, 208L); // NOTE this will change based on the data size added! - Storage.close(storage); + MakeStorage.close(storage); - storage = Storage.open(TEST_DB); + storage = MakeStorage.Disk() + .file(TEST_DB) + .serializer(manager) + .make(); int numberOfRecords2 = storage.size(); long size2 = storage.getFileSize(); Assert.assertEquals("Record count is not the same", numberOfRecords1, numberOfRecords2); Assert.assertEquals("size is not the same", size1, size2); - Storage.close(storage); + MakeStorage.close(storage); } - @Test - public void testAddAsOne() throws IOException, ClassNotFoundException { - int total = 100; - - try { - Storage storage = Storage.open(TEST_DB); - for (int i=0;i