diff --git a/Dorkbox-Util/src/dorkbox/util/storage/DiskStorage.java b/Dorkbox-Util/src/dorkbox/util/storage/DiskStorage.java index 4994945..1842036 100644 --- a/Dorkbox-Util/src/dorkbox/util/storage/DiskStorage.java +++ b/Dorkbox-Util/src/dorkbox/util/storage/DiskStorage.java @@ -34,7 +34,6 @@ import java.util.concurrent.locks.ReentrantLock; * Be wary of opening the database file in different JVM instances. Even with file-locks, you can corrupt the data. */ @SuppressWarnings("unused") -public class DiskStorage implements Storage { private final DelayTimer timer; private final ByteArrayWrapper defaultKey; @@ -59,18 +58,17 @@ class DiskStorage implements Storage { this.timer = null; } else { - this.timer = new DelayTimer("Storage Writer", false, new DelayTimer.Callback() { + this.timer = new DelayTimer("Storage Writer", false, new Runnable() { @Override public - void execute() { - Map actions = DiskStorage.this.actionMap; - + void run() { ReentrantLock actionLock2 = DiskStorage.this.actionLock; + Map actions; try { actionLock2.lock(); - // do a fast swap on the actionMap. + actions = DiskStorage.this.actionMap; DiskStorage.this.actionMap = new ConcurrentHashMap(); } finally { actionLock2.unlock(); @@ -207,7 +205,7 @@ class DiskStorage implements Storage { Object source = get0(key); if (source == null) { - // returned was null, so we should take value as the default + // returned was null, so we should save the default value putAndSave(key, data); return data; } @@ -412,6 +410,7 @@ class DiskStorage implements Storage { throw new RuntimeException("Unable to act on closed storage"); } + //noinspection SimplifiableIfStatement if (timer != null) { return this.timer.isWaiting(); } diff --git a/Dorkbox-Util/src/dorkbox/util/storage/MemoryStorage.java b/Dorkbox-Util/src/dorkbox/util/storage/MemoryStorage.java index 1d7459b..a31607a 100644 --- a/Dorkbox-Util/src/dorkbox/util/storage/MemoryStorage.java +++ b/Dorkbox-Util/src/dorkbox/util/storage/MemoryStorage.java @@ -22,9 +22,8 @@ import java.io.IOException; import java.util.concurrent.ConcurrentHashMap; /** - * + * Storage that is in memory only (and is not persisted to disk) */ -public class MemoryStorage implements Storage { private final ConcurrentHashMap storage; private final ByteArrayWrapper defaultKey; diff --git a/Dorkbox-Util/src/dorkbox/util/storage/Metadata.java b/Dorkbox-Util/src/dorkbox/util/storage/Metadata.java index 8be7996..f2a8262 100644 --- a/Dorkbox-Util/src/dorkbox/util/storage/Metadata.java +++ b/Dorkbox-Util/src/dorkbox/util/storage/Metadata.java @@ -23,11 +23,11 @@ import dorkbox.util.bytes.ByteArrayWrapper; import java.io.ByteArrayOutputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.io.RandomAccessFile; import java.lang.ref.WeakReference; import java.nio.channels.FileLock; -import java.util.zip.DeflaterOutputStream; -import java.util.zip.InflaterInputStream; @SuppressWarnings("unused") public @@ -81,7 +81,7 @@ class Metadata { */ static long getMetaDataPointer(int position) { - return StorageBase.FILE_HEADERS_REGION_LENGTH + INDEX_ENTRY_LENGTH * position; + return StorageBase.FILE_HEADERS_REGION_LENGTH + ((long) INDEX_ENTRY_LENGTH) * position; } /** @@ -135,8 +135,10 @@ class Metadata { FileLock lock = file.getChannel() .lock(origHeaderKeyPointer, INDEX_ENTRY_LENGTH, true); + file.seek(origHeaderKeyPointer); file.readFully(buf); + lock.release(); Metadata r = new Metadata(ByteArrayWrapper.wrap(buf)); @@ -145,10 +147,12 @@ class Metadata { long recordHeaderPointer = Metadata.getDataPointer(position); lock = file.getChannel() .lock(origHeaderKeyPointer, KEY_SIZE, true); + file.seek(recordHeaderPointer); r.dataPointer = file.readLong(); r.dataCapacity = file.readInt(); r.dataCount = file.readInt(); + lock.release(); if (r.dataPointer == 0L || r.dataCapacity == 0L || r.dataCount == 0L) { @@ -173,17 +177,19 @@ class Metadata { FileLock lock = file.getChannel() .lock(recordHeaderPointer, POINTER_INFO_SIZE, false); + file.seek(recordHeaderPointer); file.writeLong(this.dataPointer); file.writeInt(this.dataCapacity); file.writeInt(this.dataCount); + lock.release(); } /** * Move a record to the new INDEX */ - void move(RandomAccessFile file, int newIndex) throws IOException { + void moveRecord(RandomAccessFile file, int newIndex) throws IOException { byte[] buf = new byte[KEY_SIZE]; long origHeaderKeyPointer = Metadata.getMetaDataPointer(this.indexPosition); @@ -192,6 +198,7 @@ class Metadata { file.seek(origHeaderKeyPointer); file.readFully(buf); + lock.release(); long newHeaderKeyPointer = Metadata.getMetaDataPointer(newIndex); @@ -200,6 +207,7 @@ class Metadata { file.seek(newHeaderKeyPointer); file.write(buf); + lock.release(); // System.err.println("updating ptr: " + this.indexPosition + " -> " + newIndex + " @ " + newHeaderKeyPointer + "-" + (newHeaderKeyPointer+INDEX_ENTRY_LENGTH)); @@ -230,6 +238,7 @@ class Metadata { // save the data file.seek(position); file.write(data); + lock.release(); // update header pointer info @@ -249,6 +258,7 @@ class Metadata { .lock(this.dataPointer, this.dataCount, true); file.seek(this.dataPointer); file.readFully(buf); + lock.release(); // Sys.printArray(buf, buf.length, false, 0); @@ -260,30 +270,32 @@ class Metadata { */ static - T readData(SerializationManager serializationManager, InflaterInputStream inputStream) throws IOException { - Input input = new Input(inputStream, 1024); // read 1024 at a time + T readData(final SerializationManager serializationManager, final Input input) throws IOException { + // this is to reset the internal buffer of 'input' + input.setInputStream(input.getInputStream()); + @SuppressWarnings("unchecked") T readObject = (T) serializationManager.readFullClassAndObject(null, input); return readObject; } /** - * Writes data to the end of the file (which is where the datapointer is at) + * Writes data to the end of the file (which is where the datapointer is at). This must be locked/released in calling methods! */ static - void writeDataFast(final SerializationManager serializationManager, - final Object data, - final RandomAccessFile file, - final DeflaterOutputStream outputStream) throws IOException { - // HAVE TO LOCK BEFORE THIS IS CALLED! (AND FREE AFTERWARDS!) - Output output = new Output(outputStream, 1024); // write 1024 at a time + int writeData(final SerializationManager serializationManager, + final Object data, + final Output output) throws IOException { + + output.clear(); + serializationManager.writeFullClassAndObject(null, output, data); output.flush(); - outputStream.flush(); // sync-flush is enabled, so the output stream will finish compressing data. + return (int) output.total(); } - void writeData(ByteArrayOutputStream byteArrayOutputStream, RandomAccessFile file) throws IOException { + void writeDataRaw(ByteArrayOutputStream byteArrayOutputStream, RandomAccessFile file) throws IOException { this.dataCount = byteArrayOutputStream.size(); FileLock lock = file.getChannel() @@ -292,7 +304,8 @@ class Metadata { FileOutputStream out = new FileOutputStream(file.getFD()); file.seek(this.dataPointer); byteArrayOutputStream.writeTo(out); - out.flush(); + + lock.release(); } diff --git a/Dorkbox-Util/src/dorkbox/util/storage/Storage.java b/Dorkbox-Util/src/dorkbox/util/storage/Storage.java index 98138eb..48a5b18 100644 --- a/Dorkbox-Util/src/dorkbox/util/storage/Storage.java +++ b/Dorkbox-Util/src/dorkbox/util/storage/Storage.java @@ -61,28 +61,31 @@ interface Storage { *

* This will check to see if there is an associated key for that data, if not - it will use data as the default * - * @param data The data that will hold the copy of the data from disk + * @param data This is the default value, and if there is no value with the key in the DB this default value will be saved. */ T getAndPut(T data) throws IOException; /** * Returns the saved data for the specified key. * - * @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db) + * @param key The key used to check if data already exists. + * @param data This is the default value, and if there is no value with the key in the DB this default value will be saved. */ T getAndPut(String key, T data) throws IOException; /** * Returns the saved data for the specified key. * - * @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db) + * @param key The key used to check if data already exists. + * @param data This is the default value, and if there is no value with the key in the DB this default value will be saved. */ T getAndPut(byte[] key, T data) throws IOException; /** * Returns the saved data for the specified key. * - * @param data If there is no object in the DB with the specified key, this value will be the default (and will be saved to the db) + * @param key The key used to check if data already exists. + * @param data This is the default value, and if there is no value with the key in the DB this default value will be saved. */ @SuppressWarnings("unchecked") T getAndPut(ByteArrayWrapper key, T data) throws IOException; diff --git a/Dorkbox-Util/src/dorkbox/util/storage/StorageBase.java b/Dorkbox-Util/src/dorkbox/util/storage/StorageBase.java index 2fee373..3259b16 100644 --- a/Dorkbox-Util/src/dorkbox/util/storage/StorageBase.java +++ b/Dorkbox-Util/src/dorkbox/util/storage/StorageBase.java @@ -16,24 +16,29 @@ package dorkbox.util.storage; import com.esotericsoftware.kryo.KryoException; +import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import dorkbox.util.SerializationManager; import dorkbox.util.bytes.ByteArrayWrapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.RandomAccessFile; import java.lang.ref.WeakReference; +import java.nio.channels.Channels; import java.nio.channels.FileLock; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantLock; -import java.util.zip.Deflater; -import java.util.zip.DeflaterOutputStream; -import java.util.zip.Inflater; -import java.util.zip.InflaterInputStream; // a note on file locking between c and java @@ -42,7 +47,6 @@ import java.util.zip.InflaterInputStream; @SuppressWarnings("unused") -public class StorageBase { private final Logger logger = LoggerFactory.getLogger(getClass().getSimpleName()); @@ -70,8 +74,9 @@ class StorageBase { private final ReentrantLock referenceLock = new ReentrantLock(); + // file/raf that are used private final File baseFile; - private final RandomAccessFile file; + private final RandomAccessFile randomAccessFile; /** @@ -93,11 +98,12 @@ class StorageBase { // save references to these, so they don't have to be created/destroyed any time there is I/O private final SerializationManager serializationManager; - private final Deflater deflater; - private final DeflaterOutputStream outputStream; + private final Output output; + private final Input input; + + // input/output write buffer size before flushing to/from the file + public static final int BUFFER_SIZE = 1024; - private final Inflater inflater; - private final InflaterInputStream inputStream; /** * Creates or opens a new database file. @@ -109,65 +115,75 @@ class StorageBase { this.baseFile = filePath; - File parentFile = this.baseFile.getParentFile(); - if (parentFile != null && !parentFile.exists()) { - if (!parentFile.mkdirs()) { - throw new IOException("Unable to create dirs for: " + filePath); + boolean newStorage = !filePath.exists(); + + if (newStorage) { + File parentFile = this.baseFile.getParentFile(); + if (parentFile != null && !parentFile.exists()) { + if (!parentFile.mkdirs()) { + throw new IOException("Unable to create dirs for: " + filePath); + } } } - this.file = new RandomAccessFile(this.baseFile, "rw"); + this.randomAccessFile = new RandomAccessFile(this.baseFile, "rw"); - if (this.file.length() > FILE_HEADERS_REGION_LENGTH) { - this.file.seek(VERSION_HEADER_LOCATION); - this.databaseVersion = this.file.readInt(); - this.numberOfRecords = this.file.readInt(); - this.dataPosition = this.file.readLong(); + + if (newStorage || this.randomAccessFile.length() <= FILE_HEADERS_REGION_LENGTH) { + setVersion(this.randomAccessFile, 0); + setRecordCount(this.randomAccessFile, 0); + + // pad the metadata with 21 records, so there is about 1k of padding before the data starts + long indexPointer = Metadata.getMetaDataPointer(21); + setDataStartPosition(indexPointer); + // have to make sure we can read header info (even if it's blank) + this.randomAccessFile.setLength(indexPointer); } else { - setVersionNumber(this.file, 0); + this.randomAccessFile.seek(VERSION_HEADER_LOCATION); + this.databaseVersion = this.randomAccessFile.readInt(); + this.numberOfRecords = this.randomAccessFile.readInt(); + this.dataPosition = this.randomAccessFile.readLong(); - // always start off with 4 records - setRecordCount(this.file, 4); - - long indexPointer = Metadata.getMetaDataPointer(4); - setDataPosition(this.file, indexPointer); - // have to make sure we can read header info (even if it's blank) - this.file.setLength(indexPointer); - } - - if (this.file.length() < this.dataPosition) { - this.logger.error("Corrupted storage file!"); - throw new IllegalArgumentException("Unable to parse header information from storage. Maybe it's corrupted?"); + if (this.randomAccessFile.length() < this.dataPosition) { + this.logger.error("Corrupted storage file!"); + throw new IllegalArgumentException("Unable to parse header information from storage. Maybe it's corrupted?"); + } } //noinspection AutoBoxing this.logger.info("Storage version: {}", this.databaseVersion); - this.deflater = new Deflater(7, true); - FileOutputStream fileOutputStream = new FileOutputStream(this.file.getFD()); - this.outputStream = new DeflaterOutputStream(fileOutputStream, this.deflater, 65536); - this.inflater = new Inflater(true); - this.inputStream = new InflaterInputStream(new FileInputStream(this.file.getFD()), this.inflater, 65536); + // If we want to use compression (no need really, since this file is small already), + // then we have to make sure it's sync'd on flush AND have actually call outputStream.flush(). + final InputStream inputStream = Channels.newInputStream(randomAccessFile.getChannel()); + final OutputStream outputStream = Channels.newOutputStream(randomAccessFile.getChannel()); + + // read/write 1024 bytes at a time + output = new Output(outputStream, BUFFER_SIZE); + input = new Input(inputStream, BUFFER_SIZE); + //noinspection AutoBoxing this.weight = 0.5F; this.memoryIndex = new ConcurrentHashMap(this.numberOfRecords); - Metadata meta; - for (int index = 0; index < this.numberOfRecords; index++) { - meta = Metadata.readHeader(this.file, index); - if (meta == null) { - // because we guarantee that empty metadata are AWLAYS at the end of the section, if we get a null one, break! - break; + if (!newStorage) { + Metadata meta; + for (int index = 0; index < this.numberOfRecords; index++) { + meta = Metadata.readHeader(this.randomAccessFile, index); + if (meta == null) { + // because we guarantee that empty metadata are ALWAYS at the end of the section, if we get a null one, break! + break; + } + this.memoryIndex.put(meta.key, meta); } - this.memoryIndex.put(meta.key, meta); - } - if (this.memoryIndex.size() != this.numberOfRecords) { - setRecordCount(this.file, this.memoryIndex.size()); - this.logger.warn("Mismatch record count in storage, auto-correcting size."); + if (this.memoryIndex.size() != this.numberOfRecords) { + setRecordCount(this.randomAccessFile, this.memoryIndex.size()); + this.logger.warn("Mismatch record count in storage, auto-correcting size."); + } } } @@ -255,11 +271,12 @@ class StorageBase { try { - // else, we have to load it from disk - this.inflater.reset(); - this.file.seek(meta.dataPointer); +// System.err.println("--Reading data from: " + meta.dataPointer); - T readRecordData = Metadata.readData(this.serializationManager, this.inputStream); + // else, we have to load it from disk + this.randomAccessFile.seek(meta.dataPointer); + + T readRecordData = Metadata.readData(this.serializationManager, this.input); if (readRecordData != null) { // now stuff it into our reference cache for future lookups! @@ -310,13 +327,16 @@ class StorageBase { void close() { // pending ops flushed (protected by lock) // not protected by lock + + this.logger.info("Closing storage file: '{}'", this.baseFile.getAbsolutePath()); + try { - this.file.getFD() - .sync(); - this.file.close(); + this.randomAccessFile.getFD() + .sync(); + this.input.close(); + this.randomAccessFile.close(); this.memoryIndex.clear(); - this.inputStream.close(); } catch (IOException e) { this.logger.error("Error while closing the file", e); } @@ -330,7 +350,7 @@ class StorageBase { long getFileSize() { // protected by actionLock try { - return this.file.length(); + return this.randomAccessFile.length(); } catch (IOException e) { this.logger.error("Error getting file size for {}", this.baseFile.getAbsolutePath(), e); return -1L; @@ -355,9 +375,7 @@ class StorageBase { * Will also save the object in a cache. */ private - void save0(ByteArrayWrapper key, Object object, DeflaterOutputStream fileOutputStream, Deflater deflater) { - deflater.reset(); - + void save0(ByteArrayWrapper key, Object object) { Metadata metaData = this.memoryIndex.get(key); int currentRecordCount = this.numberOfRecords; @@ -367,29 +385,31 @@ class StorageBase { if (currentRecordCount == 1) { // if we are the ONLY one, then we can do things differently. // just dump the data again to disk. - FileLock lock = this.file.getChannel() - .lock(this.dataPosition, + FileLock lock = this.randomAccessFile.getChannel() + .lock(this.dataPosition, Long.MAX_VALUE - this.dataPosition, false); // don't know how big it is, so max value it - this.file.seek(this.dataPosition); // this is the end of the file, we know this ahead-of-time - Metadata.writeDataFast(this.serializationManager, object, file, fileOutputStream); + + this.randomAccessFile.seek(this.dataPosition); // this is the end of the file, we know this ahead-of-time + Metadata.writeData(this.serializationManager, object, this.output); // have to re-specify the capacity and size //noinspection NumericCastThatLosesPrecision - int sizeOfWrittenData = (int) (this.file.length() - this.dataPosition); + int sizeOfWrittenData = (int) (this.randomAccessFile.length() - this.dataPosition); metaData.dataCapacity = sizeOfWrittenData; metaData.dataCount = sizeOfWrittenData; + lock.release(); } else { // this is comparatively slow, since we serialize it first to get the size, then we put it in the file. - ByteArrayOutputStream dataStream = getDataAsByteArray(this.serializationManager, this.logger, object, deflater); + ByteArrayOutputStream dataStream = getDataAsByteArray(this.serializationManager, this.logger, object); int size = dataStream.size(); if (size > metaData.dataCapacity) { deleteRecordData(metaData, size); // stuff this record to the end of the file, since it won't fit in it's current location - metaData.dataPointer = this.file.length(); + metaData.dataPointer = this.randomAccessFile.length(); // have to make sure that the CAPACITY of the new one is the SIZE of the new data! // and since it is going to the END of the file, we do that. metaData.dataCapacity = size; @@ -398,45 +418,51 @@ class StorageBase { // TODO: should check to see if the data is different. IF SO, then we write, otherwise nothing! - metaData.writeData(dataStream, this.file); + metaData.writeDataRaw(dataStream, this.randomAccessFile); } - metaData.writeDataInfo(this.file); + metaData.writeDataInfo(this.randomAccessFile); } catch (IOException e) { this.logger.error("Error while saving data to disk", e); } } else { + // metadata == null... try { - // try to move the read head in order - setRecordCount(this.file, currentRecordCount + 1); + // set the number of records that this storage has + setRecordCount(this.randomAccessFile, currentRecordCount + 1); // This will make sure that there is room to write a new record. This is zero indexed. // this will skip around if moves occur - ensureIndexCapacity(this.file); + ensureIndexCapacity(this.randomAccessFile); // append record to end of file - long length = this.file.length(); + long length = this.randomAccessFile.length(); + +// System.err.println("--Writing data to: " + length); + metaData = new Metadata(key, currentRecordCount, length); - metaData.writeMetaDataInfo(this.file); + metaData.writeMetaDataInfo(this.randomAccessFile); // add new entry to the index this.memoryIndex.put(key, metaData); - setRecordCount(this.file, currentRecordCount + 1); // save out the data. Because we KNOW that we are writing this to the end of the file, // there are some tricks we can use. - FileLock lock = this.file.getChannel() - .lock(length, Long.MAX_VALUE - length, false); // don't know how big it is, so max value it - this.file.seek(length); // this is the end of the file, we know this ahead-of-time - Metadata.writeDataFast(this.serializationManager, object, file, fileOutputStream); + // don't know how big it is, so max value it + FileLock lock = this.randomAccessFile.getChannel() + .lock(0, Long.MAX_VALUE, false); + + // this is the end of the file, we know this ahead-of-time + this.randomAccessFile.seek(length); + + int total = Metadata.writeData(this.serializationManager, object, this.output); lock.release(); - metaData.dataCount = deflater.getTotalOut(); - metaData.dataCapacity = metaData.dataCount; + metaData.dataCount = metaData.dataCapacity = total; // have to save it. - metaData.writeDataInfo(this.file); + metaData.writeDataInfo(this.randomAccessFile); } catch (IOException e) { this.logger.error("Error while writing data to disk", e); return; @@ -449,32 +475,31 @@ class StorageBase { private static - ByteArrayOutputStream getDataAsByteArray(SerializationManager serializationManager, Logger logger, Object data, Deflater deflater) throws IOException { - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - OutputStream outputStream = new DeflaterOutputStream(byteArrayOutputStream, deflater); + ByteArrayOutputStream getDataAsByteArray(SerializationManager serializationManager, Logger logger, Object data) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); Output output = new Output(outputStream, 1024); // write 1024 at a time + serializationManager.writeFullClassAndObject(logger, output, data); output.flush(); outputStream.flush(); outputStream.close(); - return byteArrayOutputStream; + return outputStream; } void doActionThings(Map actions) { - DeflaterOutputStream outputStream2 = this.outputStream; - Deflater deflater2 = this.deflater; // actions is thrown away after this invocation. GC can pick it up. // we are only interested in the LAST action that happened for some data. // items to be "autosaved" are automatically injected into "actions". - for (Entry entry : actions.entrySet()) { + final Set> entries = actions.entrySet(); + for (Entry entry : entries) { Object object = entry.getValue(); ByteArrayWrapper key = entry.getKey(); // our action list is for explicitly saving objects (but not necessarily "registering" them to be auto-saved - save0(key, object, outputStream2, deflater2); + save0(key, object); } } @@ -501,11 +526,11 @@ class StorageBase { private void deleteRecordData(Metadata deletedRecord, int sizeOfDataToAdd) throws IOException { - if (this.file.length() == deletedRecord.dataPointer + deletedRecord.dataCapacity) { + if (this.randomAccessFile.length() == deletedRecord.dataPointer + deletedRecord.dataCapacity) { // shrink file since this is the last record in the file - FileLock lock = this.file.getChannel() - .lock(deletedRecord.dataPointer, Long.MAX_VALUE - deletedRecord.dataPointer, false); - this.file.setLength(deletedRecord.dataPointer); + FileLock lock = this.randomAccessFile.getChannel() + .lock(deletedRecord.dataPointer, Long.MAX_VALUE - deletedRecord.dataPointer, false); + this.randomAccessFile.setLength(deletedRecord.dataPointer); lock.release(); } else { @@ -527,11 +552,11 @@ class StorageBase { if (endIndexPointer < this.dataPosition && endIndexPointer <= newEndOfDataPointer) { // one option is to shrink the RECORD section to fit the new data - setDataPosition(this.file, newEndOfDataPointer); + setDataStartPosition(newEndOfDataPointer); } else { // option two is to grow the RECORD section, and put the data at the end of the file - setDataPosition(this.file, endOfDataPointer); + setDataStartPosition(endOfDataPointer); } } else { @@ -539,7 +564,7 @@ class StorageBase { if (previous != null) { // append space of deleted record onto previous record previous.dataCapacity += deletedRecord.dataCapacity; - previous.writeDataInfo(this.file); + previous.writeDataInfo(this.randomAccessFile); } else { // because there is no "previous", that means we MIGHT be the FIRST record @@ -556,15 +581,15 @@ class StorageBase { int currentNumRecords = this.memoryIndex.size(); if (deleteRecord.indexPosition != currentNumRecords - 1) { - Metadata last = Metadata.readHeader(this.file, currentNumRecords - 1); + Metadata last = Metadata.readHeader(this.randomAccessFile, currentNumRecords - 1); assert last != null; - last.move(this.file, deleteRecord.indexPosition); + last.moveRecord(this.randomAccessFile, deleteRecord.indexPosition); } this.memoryIndex.remove(key); - setRecordCount(this.file, currentNumRecords - 1); + setRecordCount(this.randomAccessFile, currentNumRecords - 1); } @@ -572,13 +597,14 @@ class StorageBase { * Writes the number of records header to the file. */ private - void setVersionNumber(RandomAccessFile file, int versionNumber) throws IOException { + void setVersion(RandomAccessFile file, int versionNumber) throws IOException { this.databaseVersion = versionNumber; - FileLock lock = this.file.getChannel() - .lock(VERSION_HEADER_LOCATION, 4, false); + FileLock lock = this.randomAccessFile.getChannel() + .lock(VERSION_HEADER_LOCATION, 4, false); file.seek(VERSION_HEADER_LOCATION); file.writeInt(versionNumber); + lock.release(); } @@ -587,26 +613,34 @@ class StorageBase { */ private void setRecordCount(RandomAccessFile file, int numberOfRecords) throws IOException { - this.numberOfRecords = numberOfRecords; + if (this.numberOfRecords != numberOfRecords) { + this.numberOfRecords = numberOfRecords; - FileLock lock = this.file.getChannel() - .lock(NUM_RECORDS_HEADER_LOCATION, 4, false); - file.seek(NUM_RECORDS_HEADER_LOCATION); - file.writeInt(numberOfRecords); - lock.release(); +// System.err.println("Set recordCount: " + numberOfRecords); + + FileLock lock = this.randomAccessFile.getChannel() + .lock(NUM_RECORDS_HEADER_LOCATION, 4, false); + file.seek(NUM_RECORDS_HEADER_LOCATION); + file.writeInt(numberOfRecords); + + lock.release(); + } } /** * Writes the data start position to the file. */ private - void setDataPosition(RandomAccessFile file, long dataPositionPointer) throws IOException { - this.dataPosition = dataPositionPointer; + void setDataStartPosition(long dataPositionPointer) throws IOException { + FileLock lock = this.randomAccessFile.getChannel() + .lock(DATA_START_HEADER_LOCATION, 8, false); + +// System.err.println("Setting data position: " + dataPositionPointer); + dataPosition = dataPositionPointer; + + randomAccessFile.seek(DATA_START_HEADER_LOCATION); + randomAccessFile.writeLong(dataPositionPointer); - FileLock lock = this.file.getChannel() - .lock(DATA_START_HEADER_LOCATION, 8, false); - file.seek(DATA_START_HEADER_LOCATION); - file.writeLong(dataPositionPointer); lock.release(); } @@ -616,7 +650,7 @@ class StorageBase { void setVersion(int versionNumber) { try { - setVersionNumber(this.file, versionNumber); + setVersion(this.randomAccessFile, versionNumber); } catch (IOException e) { this.logger.error("Unable to set the version number", e); } @@ -650,14 +684,16 @@ class StorageBase { */ private void ensureIndexCapacity(RandomAccessFile file) throws IOException { - int numberOfRecords = this.numberOfRecords; // because we are zero indexed, this is ALSO the index where the record will START - long endIndexPointer = Metadata.getMetaDataPointer( - numberOfRecords + 1); // +1 because this is where that index will END (the start of the NEXT one) + // because we are zero indexed, this is ALSO the index where the record will START + int numberOfRecords = this.numberOfRecords; + + // +1 because this is where that index will END (the start of the NEXT one) + long endIndexPointer = Metadata.getMetaDataPointer(numberOfRecords + 1); // just set the data position to the end of the file, since we don't have any data yet. if (endIndexPointer > file.length() && numberOfRecords == 0) { file.setLength(endIndexPointer); - setDataPosition(file, endIndexPointer); + setDataStartPosition(endIndexPointer); return; } @@ -675,13 +711,21 @@ class StorageBase { int newNumberOfRecords = getWeightedNewRecordCount(numberOfRecords); endIndexPointer = Metadata.getMetaDataPointer(newNumberOfRecords); + // sometimes the endIndexPointer is in the middle of data, so we cannot move a record to where // data already exists, we have to move it to the end. Since we GUARANTEE that there is never "free space" at the // end of a file, this is ok - endIndexPointer = Math.max(endIndexPointer, file.length()); + if (endIndexPointer > file.length()) { + // make sure we adjust the file size + file.setLength(endIndexPointer); + } + else { + endIndexPointer = file.length(); + } // we know that the start of the NEW data position has to be here. - setDataPosition(file, endIndexPointer); + setDataStartPosition(endIndexPointer); + long writeDataPosition = endIndexPointer; diff --git a/Dorkbox-Util/src/dorkbox/util/storage/Store.java b/Dorkbox-Util/src/dorkbox/util/storage/Store.java index 157a163..1031198 100644 --- a/Dorkbox-Util/src/dorkbox/util/storage/Store.java +++ b/Dorkbox-Util/src/dorkbox/util/storage/Store.java @@ -32,6 +32,21 @@ class Store { @SuppressWarnings("SpellCheckingInspection") private static final Map storages = new HashMap(1); + // Make sure that the timer is run on shutdown. A HARD shutdown will just POW! kill it, a "nice" shutdown will run the hook + private static Thread shutdownHook = new Thread(new Runnable() { + @Override + public + void run() { + Store.shutdown(); + } + }); + + static { + // add a shutdown hook to make sure that we properly flush/shutdown storage. + Runtime.getRuntime() + .addShutdownHook(shutdownHook); + } + public static DiskMaker Disk() { return new DiskMaker(); diff --git a/Dorkbox-Util/test/dorkbox/util/StorageTest.java b/Dorkbox-Util/test/dorkbox/util/StorageTest.java index 75fdecb..e81af44 100644 --- a/Dorkbox-Util/test/dorkbox/util/StorageTest.java +++ b/Dorkbox-Util/test/dorkbox/util/StorageTest.java @@ -18,6 +18,13 @@ import java.util.Arrays; @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class StorageTest { + static int total = 10; + // the initial size is specified during disk.storage construction, and is based on the number of padded records. + private static final long initialSize = 1024L; + + // this is the size for each record (determined by looking at the output when writing the file) + private static final int sizePerRecord = 23; + private static final File TEST_DB = new File("sampleFile.records"); private static final SerializationManager manager = new SerializationManager() { @@ -126,7 +133,7 @@ class StorageTest { long size1 = storage.getFileSize(); Assert.assertEquals("count is not correct", numberOfRecords1, 0); - Assert.assertEquals("size is not correct", size1, 208L); // NOTE this will change based on the data size added! + Assert.assertEquals("size is not correct", size1, initialSize); Store.close(storage); @@ -134,6 +141,7 @@ class StorageTest { .file(TEST_DB) .serializer(manager) .make(); + int numberOfRecords2 = storage.size(); long size2 = storage.getFileSize(); @@ -147,13 +155,12 @@ class StorageTest { @Test public void testAddAsOne() throws IOException, ClassNotFoundException { - int total = 100; - try { Storage storage = Store.Disk() .file(TEST_DB) .serializer(manager) .make(); + for (int i = 0; i < total; i++) { add(storage, i); } @@ -163,6 +170,7 @@ class StorageTest { .file(TEST_DB) .serializer(manager) .make(); + for (int i = 0; i < total; i++) { String record1Data = createData(i); String readRecord = readRecord(storage, i); @@ -177,16 +185,20 @@ class StorageTest { } } + /** + * Adds data to storage using the SAME key each time (so each entry is overwritten). + * @throws IOException + * @throws ClassNotFoundException + */ @Test public void testAddNoKeyRecords() throws IOException, ClassNotFoundException { - int total = 100; - try { Storage storage = Store.Disk() .file(TEST_DB) .serializer(manager) .make(); + for (int i = 0; i < total; i++) { log("adding record " + i + "..."); String addRecord = createData(i); @@ -208,13 +220,15 @@ class StorageTest { log("reading record " + (total - 1) + "..."); String readData = storage.get(); + // the ONLY entry in storage should be the last one that we added Assert.assertEquals("Object is not the same", dataCheck, readData); int numberOfRecords1 = storage.size(); long size1 = storage.getFileSize(); Assert.assertEquals("count is not correct", numberOfRecords1, 1); - Assert.assertEquals("size is not correct", size1, 235L); // NOTE this will change based on the data size added! + + Assert.assertEquals("size is not correct", size1, initialSize + sizePerRecord); Store.close(storage); } catch (Exception e) { @@ -226,13 +240,12 @@ class StorageTest { @Test public void testAddRecords_DelaySaveA() throws IOException, ClassNotFoundException { - int total = 100; - try { Storage storage = Store.Disk() .file(TEST_DB) .serializer(manager) .make(); + for (int i = 0; i < total; i++) { add(storage, i); } @@ -272,13 +285,12 @@ class StorageTest { @Test public void testAddRecords_DelaySaveB() throws IOException, ClassNotFoundException { - int total = 100; - try { Storage storage = Store.Disk() .file(TEST_DB) .serializer(manager) .make(); + for (int i = 0; i < total; i++) { add(storage, i); } @@ -296,6 +308,7 @@ class StorageTest { .file(TEST_DB) .serializer(manager) .make(); + for (int i = 0; i < total; i++) { String dataCheck = createData(i); String readRecord = readRecord(storage, i); @@ -313,13 +326,12 @@ class StorageTest { @Test public void testLoadRecords() throws IOException, ClassNotFoundException { - int total = 100; - try { Storage storage = Store.Disk() .file(TEST_DB) .serializer(manager) .make(); + for (int i = 0; i < total; i++) { String addRecord = add(storage, i); String readRecord = readRecord(storage, i); @@ -332,6 +344,7 @@ class StorageTest { .file(TEST_DB) .serializer(manager) .make(); + for (int i = 0; i < total; i++) { String dataCheck = createData(i); String readRecord = readRecord(storage, i); @@ -346,8 +359,8 @@ class StorageTest { storage.put(createKey, data); - Data data2 = new Data(); - storage.getAndPut(createKey, data2); + Data data2; + data2 = storage.getAndPut(createKey, new Data()); Assert.assertEquals("Object is not the same", data, data2); Store.close(storage); @@ -356,8 +369,7 @@ class StorageTest { .serializer(manager) .make(); - data2 = new Data(); - storage.getAndPut(createKey, data2); + data2 = storage.getAndPut(createKey, new Data()); Assert.assertEquals("Object is not the same", data, data2); Store.close(storage); @@ -371,13 +383,16 @@ class StorageTest { @Test public void testAddRecordsDelete1Record() throws IOException, ClassNotFoundException { - int total = 100; + if (total < 4) { + throw new IOException("Unable to run test with too few entries."); + } try { Storage storage = Store.Disk() .file(TEST_DB) .serializer(manager) .make(); + for (int i = 0; i < total; i++) { String addRecord = add(storage, i); String readRecord = readRecord(storage, i); @@ -390,6 +405,7 @@ class StorageTest { .file(TEST_DB) .serializer(manager) .make(); + for (int i = 0; i < total; i++) { String dataCheck = createData(i); String readRecord = readRecord(storage, i); @@ -442,13 +458,12 @@ class StorageTest { @Test public void testUpdateRecords() throws IOException, ClassNotFoundException { - int total = 100; - try { Storage storage = Store.Disk() .file(TEST_DB) .serializer(manager) .make(); + for (int i = 0; i < total; i++) { String addRecord = add(storage, i); String readRecord = readRecord(storage, i); @@ -507,13 +522,12 @@ class StorageTest { @Test public void testSaveAllRecords() throws IOException, ClassNotFoundException { - int total = 100; - try { Storage storage = Store.Disk() .file(TEST_DB) .serializer(manager) .make(); + for (int i = 0; i < total; i++) { Data data = new Data(); makeData(data); @@ -533,8 +547,8 @@ class StorageTest { for (int i = 0; i < total; i++) { String createKey = createKey(i); - Data data2 = new Data(); - storage.getAndPut(createKey, data2); + Data data2; + data2 = storage.getAndPut(createKey, new Data()); Assert.assertEquals("Object is not the same", data, data2); } Store.close(storage); @@ -607,19 +621,19 @@ class StorageTest { data.strings = new String[] {"ab012", "", null, "!@#$", "�����"}; data.ints = new int[] {-1234567, 1234567, -1, 0, 1, Integer.MAX_VALUE, Integer.MIN_VALUE}; - data.shorts = new short[] {-12345, 12345, -1, 0, 1, Short.MAX_VALUE, Short.MIN_VALUE}; + data.shorts = new short[] {(short) -12345, (short) 12345, (short) -1, (short) 0, (short) 1, Short.MAX_VALUE, Short.MIN_VALUE}; data.floats = new float[] {0, -0, 1, -1, 123456, -123456, 0.1f, 0.2f, -0.3f, (float) Math.PI, Float.MAX_VALUE, Float.MIN_VALUE}; data.doubles = new double[] {0, -0, 1, -1, 123456, -123456, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE, Double.MIN_VALUE}; - data.longs = new long[] {0, -0, 1, -1, 123456, -123456, 99999999999l, -99999999999l, Long.MAX_VALUE, Long.MIN_VALUE}; - data.bytes = new byte[] {-123, 123, -1, 0, 1, Byte.MAX_VALUE, Byte.MIN_VALUE}; + data.longs = new long[] {0, -0, 1, -1, 123456, -123456, 99999999999L, -99999999999L, Long.MAX_VALUE, Long.MIN_VALUE}; + data.bytes = new byte[] {(byte) -123, (byte) 123, (byte) -1, (byte) 0, (byte) 1, Byte.MAX_VALUE, Byte.MIN_VALUE}; data.chars = new char[] {32345, 12345, 0, 1, 63, Character.MAX_VALUE, Character.MIN_VALUE}; data.booleans = new boolean[] {true, false}; data.Ints = new Integer[] {-1234567, 1234567, -1, 0, 1, Integer.MAX_VALUE, Integer.MIN_VALUE}; data.Shorts = new Short[] {-12345, 12345, -1, 0, 1, Short.MAX_VALUE, Short.MIN_VALUE}; - data.Floats = new Float[] {0f, -0f, 1f, -1f, 123456f, -123456f, 0.1f, 0.2f, -0.3f, (float) Math.PI, Float.MAX_VALUE, + data.Floats = new Float[] {0.0f, -0.0f, 1.0f, -1.0f, 123456.0f, -123456.0f, 0.1f, 0.2f, -0.3f, (float) Math.PI, Float.MAX_VALUE, Float.MIN_VALUE}; - data.Doubles = new Double[] {0d, -0d, 1d, -1d, 123456d, -123456d, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE, Double.MIN_VALUE}; - data.Longs = new Long[] {0l, -0l, 1l, -1l, 123456l, -123456l, 99999999999l, -99999999999l, Long.MAX_VALUE, Long.MIN_VALUE}; + data.Doubles = new Double[] {0.0d, -0.0d, 1.0d, -1.0d, 123456.0d, -123456.0d, 0.1d, 0.2d, -0.3d, Math.PI, Double.MAX_VALUE, Double.MIN_VALUE}; + data.Longs = new Long[] {0L, -0L, 1L, -1L, 123456L, -123456L, 99999999999L, -99999999999L, Long.MAX_VALUE, Long.MIN_VALUE}; data.Bytes = new Byte[] {-123, 123, -1, 0, 1, Byte.MAX_VALUE, Byte.MIN_VALUE}; data.Chars = new Character[] {32345, 12345, 0, 1, 63, Character.MAX_VALUE, Character.MIN_VALUE}; data.Booleans = new Boolean[] {true, false};