Converted storage to use the 'single writer principle' for read speed
This commit is contained in:
parent
85a745dc3a
commit
99b95074ab
@ -17,11 +17,11 @@ package dorkbox.util.storage;
|
|||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.util.HashMap;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
import java.util.concurrent.atomic.AtomicBoolean;
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
import java.util.concurrent.locks.ReentrantLock;
|
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
|
|
||||||
@ -38,14 +38,23 @@ import dorkbox.util.SerializationManager;
|
|||||||
class DiskStorage implements Storage {
|
class DiskStorage implements Storage {
|
||||||
// null if we are a read-only storage
|
// null if we are a read-only storage
|
||||||
private final DelayTimer timer;
|
private final DelayTimer timer;
|
||||||
|
|
||||||
|
|
||||||
|
// must be volatile
|
||||||
|
private volatile HashMap<StorageKey, Object> actionMap = new HashMap<StorageKey, Object>();
|
||||||
|
|
||||||
|
private final Object singleWriterLock = new Object[0];
|
||||||
|
|
||||||
|
// Recommended for best performance while adhering to the "single writer principle". Must be static-final
|
||||||
|
private static final AtomicReferenceFieldUpdater<DiskStorage, HashMap> actionMapREF =
|
||||||
|
AtomicReferenceFieldUpdater.newUpdater(DiskStorage.class, HashMap.class, "actionMap");
|
||||||
|
|
||||||
private final StorageBase storage;
|
private final StorageBase storage;
|
||||||
|
|
||||||
private final AtomicInteger references = new AtomicInteger(1);
|
private final AtomicInteger references = new AtomicInteger(1);
|
||||||
private final ReentrantLock actionLock = new ReentrantLock();
|
|
||||||
private final AtomicBoolean isOpen = new AtomicBoolean(false);
|
private final AtomicBoolean isOpen = new AtomicBoolean(false);
|
||||||
private volatile long milliSeconds = 3000L;
|
private volatile long milliSeconds = 3000L;
|
||||||
|
|
||||||
private volatile Map<StorageKey, Object> actionMap = new ConcurrentHashMap<StorageKey, Object>();
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates or opens a new database file.
|
* Creates or opens a new database file.
|
||||||
@ -61,16 +70,14 @@ class DiskStorage implements Storage {
|
|||||||
@Override
|
@Override
|
||||||
public
|
public
|
||||||
void run() {
|
void run() {
|
||||||
ReentrantLock actionLock = DiskStorage.this.actionLock;
|
|
||||||
|
|
||||||
Map<StorageKey, Object> actions;
|
Map<StorageKey, Object> actions;
|
||||||
try {
|
|
||||||
actionLock.lock();
|
// synchronized is used here to ensure the "single writer principle", and make sure that ONLY one thread at a time can enter this
|
||||||
|
// section. Because of this, we can have unlimited reader threads all going at the same time, without contention.
|
||||||
|
synchronized (singleWriterLock) {
|
||||||
// do a fast swap on the actionMap.
|
// do a fast swap on the actionMap.
|
||||||
actions = DiskStorage.this.actionMap;
|
actions = DiskStorage.this.actionMap;
|
||||||
DiskStorage.this.actionMap = new ConcurrentHashMap<StorageKey, Object>();
|
DiskStorage.this.actionMap = new HashMap<StorageKey, Object>();
|
||||||
} finally {
|
|
||||||
actionLock.unlock();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
DiskStorage.this.storage.doActionThings(actions);
|
DiskStorage.this.storage.doActionThings(actions);
|
||||||
@ -114,8 +121,11 @@ class DiskStorage implements Storage {
|
|||||||
throw new RuntimeException("Unable to act on closed storage");
|
throw new RuntimeException("Unable to act on closed storage");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// access a snapshot of the actionMap (single-writer-principle)
|
||||||
|
final HashMap actionMap = actionMapREF.get(this);
|
||||||
|
|
||||||
// check if our pending actions has it, or if our storage index has it
|
// check if our pending actions has it, or if our storage index has it
|
||||||
return this.actionMap.containsKey(key) || this.storage.contains(key);
|
return actionMap.containsKey(key) || this.storage.contains(key);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -175,24 +185,21 @@ class DiskStorage implements Storage {
|
|||||||
throw new RuntimeException("Unable to act on closed storage");
|
throw new RuntimeException("Unable to act on closed storage");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// access a snapshot of the actionMap (single-writer-principle)
|
||||||
|
final HashMap actionMap = actionMapREF.get(this);
|
||||||
|
|
||||||
// if the object in is pending, we get it from there
|
// if the object in is pending, we get it from there
|
||||||
try {
|
Object object = actionMap.get(key);
|
||||||
this.actionLock.lock();
|
|
||||||
|
|
||||||
Object object = this.actionMap.get(key);
|
if (object != null) {
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
if (object != null) {
|
T returnObject = (T) object;
|
||||||
@SuppressWarnings("unchecked")
|
return returnObject;
|
||||||
T returnObject = (T) object;
|
|
||||||
return returnObject;
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
this.actionLock.unlock();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// not found, so we have to go find it on disk
|
// not found, so we have to go find it on disk
|
||||||
return this.storage.get(key);
|
return this.storage.get(key);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Saves the given data to storage with the associated key.
|
* Saves the given data to storage with the associated key.
|
||||||
@ -208,18 +215,11 @@ class DiskStorage implements Storage {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (timer != null) {
|
if (timer != null) {
|
||||||
try {
|
// synchronized is used here to ensure the "single writer principle", and make sure that ONLY one thread at a time can enter this
|
||||||
this.actionLock.lock();
|
// section. Because of this, we can have unlimited reader threads all going at the same time, without contention.
|
||||||
|
synchronized (singleWriterLock) {
|
||||||
if (object != null) {
|
// push action to map
|
||||||
// push action to map
|
actionMap.put(key, object);
|
||||||
this.actionMap.put(key, object);
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
this.actionMap.remove(key);
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
this.actionLock.unlock();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// timer action runs on TIMER thread, not this thread
|
// timer action runs on TIMER thread, not this thread
|
||||||
@ -243,6 +243,7 @@ class DiskStorage implements Storage {
|
|||||||
|
|
||||||
// timer action runs on THIS thread, not timer thread
|
// timer action runs on THIS thread, not timer thread
|
||||||
if (timer != null) {
|
if (timer != null) {
|
||||||
|
// flush to storage, so we know if there were errors deleting from disk
|
||||||
this.timer.delay(0L);
|
this.timer.delay(0L);
|
||||||
return this.storage.delete(key);
|
return this.storage.delete(key);
|
||||||
}
|
}
|
||||||
|
@ -16,19 +16,29 @@
|
|||||||
package dorkbox.util.storage;
|
package dorkbox.util.storage;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.HashMap;
|
||||||
|
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Storage that is in memory only (and is not persisted to disk)
|
* Storage that is in memory only (and is not persisted to disk)
|
||||||
*/
|
*/
|
||||||
class MemoryStorage implements Storage {
|
class MemoryStorage implements Storage {
|
||||||
private final ConcurrentHashMap<StorageKey, Object> storage;
|
|
||||||
|
|
||||||
|
// must be volatile
|
||||||
|
@SuppressWarnings("MismatchedQueryAndUpdateOfCollection")
|
||||||
|
private volatile HashMap<StorageKey, Object> storage = new HashMap<StorageKey, Object>();
|
||||||
|
|
||||||
|
private final Object singleWriterLock = new Object[0];
|
||||||
|
|
||||||
|
// Recommended for best performance while adhering to the "single writer principle". Must be static-final
|
||||||
|
private static final AtomicReferenceFieldUpdater<MemoryStorage, HashMap> storageREF =
|
||||||
|
AtomicReferenceFieldUpdater.newUpdater(MemoryStorage.class, HashMap.class, "storage");
|
||||||
|
|
||||||
private int version;
|
private int version;
|
||||||
|
|
||||||
|
|
||||||
MemoryStorage() {
|
MemoryStorage() {}
|
||||||
this.storage = new ConcurrentHashMap<StorageKey, Object>();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -37,6 +47,8 @@ class MemoryStorage implements Storage {
|
|||||||
@Override
|
@Override
|
||||||
public
|
public
|
||||||
int size() {
|
int size() {
|
||||||
|
// access a snapshot of the storage (single-writer-principle)
|
||||||
|
HashMap storage = storageREF.get(this);
|
||||||
return storage.size();
|
return storage.size();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -46,6 +58,8 @@ class MemoryStorage implements Storage {
|
|||||||
@Override
|
@Override
|
||||||
public
|
public
|
||||||
boolean contains(final StorageKey key) {
|
boolean contains(final StorageKey key) {
|
||||||
|
// access a snapshot of the storage (single-writer-principle)
|
||||||
|
HashMap storage = storageREF.get(this);
|
||||||
return storage.containsKey(key);
|
return storage.containsKey(key);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -56,6 +70,8 @@ class MemoryStorage implements Storage {
|
|||||||
@Override
|
@Override
|
||||||
public
|
public
|
||||||
<T> T get(final StorageKey key) {
|
<T> T get(final StorageKey key) {
|
||||||
|
// access a snapshot of the storage (single-writer-principle)
|
||||||
|
HashMap storage = storageREF.get(this);
|
||||||
return (T) storage.get(key);
|
return (T) storage.get(key);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -63,6 +79,9 @@ class MemoryStorage implements Storage {
|
|||||||
@Override
|
@Override
|
||||||
public
|
public
|
||||||
<T> T get(final StorageKey key, final T data) {
|
<T> T get(final StorageKey key, final T data) {
|
||||||
|
// access a snapshot of the storage (single-writer-principle)
|
||||||
|
HashMap storage = storageREF.get(this);
|
||||||
|
|
||||||
final Object o = storage.get(key);
|
final Object o = storage.get(key);
|
||||||
if (o == null) {
|
if (o == null) {
|
||||||
storage.put(key, data);
|
storage.put(key, data);
|
||||||
@ -80,7 +99,11 @@ class MemoryStorage implements Storage {
|
|||||||
@Override
|
@Override
|
||||||
public
|
public
|
||||||
void put(final StorageKey key, final Object object) {
|
void put(final StorageKey key, final Object object) {
|
||||||
storage.put(key, object);
|
// synchronized is used here to ensure the "single writer principle", and make sure that ONLY one thread at a time can enter this
|
||||||
|
// section. Because of this, we can have unlimited reader threads all going at the same time, without contention.
|
||||||
|
synchronized (singleWriterLock) {
|
||||||
|
storage.put(key, object);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -89,9 +112,14 @@ class MemoryStorage implements Storage {
|
|||||||
* @return true if the delete was successful. False if there were problems deleting the data.
|
* @return true if the delete was successful. False if there were problems deleting the data.
|
||||||
*/
|
*/
|
||||||
@Override
|
@Override
|
||||||
public
|
public synchronized
|
||||||
boolean delete(final StorageKey key) {
|
boolean delete(final StorageKey key) {
|
||||||
storage.remove(key);
|
// synchronized is used here to ensure the "single writer principle", and make sure that ONLY one thread at a time can enter this
|
||||||
|
// section. Because of this, we can have unlimited reader threads all going at the same time, without contention.
|
||||||
|
synchronized (singleWriterLock) {
|
||||||
|
storage.remove(key);
|
||||||
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -173,6 +201,7 @@ class MemoryStorage implements Storage {
|
|||||||
/**
|
/**
|
||||||
* In-memory storage systems do not have a backing file, so there is nothing to close
|
* In-memory storage systems do not have a backing file, so there is nothing to close
|
||||||
*/
|
*/
|
||||||
|
@Override
|
||||||
public
|
public
|
||||||
void close() {
|
void close() {
|
||||||
StorageSystem.close(this);
|
StorageSystem.close(this);
|
||||||
|
@ -24,11 +24,12 @@ import java.io.RandomAccessFile;
|
|||||||
import java.lang.ref.WeakReference;
|
import java.lang.ref.WeakReference;
|
||||||
import java.nio.channels.Channels;
|
import java.nio.channels.Channels;
|
||||||
import java.nio.channels.FileLock;
|
import java.nio.channels.FileLock;
|
||||||
|
import java.util.HashMap;
|
||||||
import java.util.Iterator;
|
import java.util.Iterator;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Map.Entry;
|
import java.util.Map.Entry;
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
|
||||||
import java.util.concurrent.locks.ReentrantLock;
|
import java.util.concurrent.locks.ReentrantLock;
|
||||||
|
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
@ -63,8 +64,17 @@ class StorageBase {
|
|||||||
static final int FILE_HEADERS_REGION_LENGTH = 16;
|
static final int FILE_HEADERS_REGION_LENGTH = 16;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
// must be volatile
|
||||||
// The in-memory index (for efficiency, all of the record info is cached in memory).
|
// The in-memory index (for efficiency, all of the record info is cached in memory).
|
||||||
private final Map<StorageKey, Metadata> memoryIndex;
|
private volatile HashMap<StorageKey, Metadata> memoryIndex;
|
||||||
|
|
||||||
|
private final Object singleWriterLock = new Object[0];
|
||||||
|
|
||||||
|
// Recommended for best performance while adhering to the "single writer principle". Must be static-final
|
||||||
|
private static final AtomicReferenceFieldUpdater<StorageBase, HashMap> memoryREF =
|
||||||
|
AtomicReferenceFieldUpdater.newUpdater(StorageBase.class, HashMap.class, "memoryIndex");
|
||||||
|
|
||||||
|
|
||||||
// determines how much the index will grow by
|
// determines how much the index will grow by
|
||||||
private final Float weight;
|
private final Float weight;
|
||||||
@ -171,25 +181,29 @@ class StorageBase {
|
|||||||
input = new Input(inputStream, BUFFER_SIZE);
|
input = new Input(inputStream, BUFFER_SIZE);
|
||||||
|
|
||||||
|
|
||||||
//noinspection AutoBoxing
|
|
||||||
this.weight = 0.5F;
|
this.weight = 0.5F;
|
||||||
this.memoryIndex = new ConcurrentHashMap<StorageKey, Metadata>(this.numberOfRecords);
|
|
||||||
|
|
||||||
if (!newStorage) {
|
// synchronized is used here to ensure the "single writer principle", and make sure that ONLY one thread at a time can enter this
|
||||||
Metadata meta;
|
// section. Because of this, we can have unlimited reader threads all going at the same time, without contention.
|
||||||
for (int index = 0; index < this.numberOfRecords; index++) {
|
synchronized (singleWriterLock) {
|
||||||
meta = Metadata.readHeader(this.randomAccessFile, index);
|
this.memoryIndex = new HashMap<StorageKey, Metadata>(this.numberOfRecords);
|
||||||
if (meta == null) {
|
|
||||||
// because we guarantee that empty metadata are ALWAYS at the end of the section, if we get a null one, break!
|
if (!newStorage) {
|
||||||
break;
|
Metadata meta;
|
||||||
|
for (int index = 0; index < this.numberOfRecords; index++) {
|
||||||
|
meta = Metadata.readHeader(this.randomAccessFile, index);
|
||||||
|
if (meta == null) {
|
||||||
|
// because we guarantee that empty metadata are ALWAYS at the end of the section, if we get a null one, break!
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
this.memoryIndex.put(meta.key, meta);
|
||||||
}
|
}
|
||||||
this.memoryIndex.put(meta.key, meta);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.memoryIndex.size() != (this.numberOfRecords)) {
|
if (this.memoryIndex.size() != (this.numberOfRecords)) {
|
||||||
setRecordCount(this.randomAccessFile, this.memoryIndex.size());
|
setRecordCount(this.randomAccessFile, this.memoryIndex.size());
|
||||||
if (logger != null) {
|
if (logger != null) {
|
||||||
logger.warn("Mismatch record count in storage, auto-correcting size.");
|
logger.warn("Mismatch record count in storage, auto-correcting size.");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -202,7 +216,10 @@ class StorageBase {
|
|||||||
int size() {
|
int size() {
|
||||||
// wrapper flushes first (protected by lock)
|
// wrapper flushes first (protected by lock)
|
||||||
// not protected by lock
|
// not protected by lock
|
||||||
return this.memoryIndex.size();
|
|
||||||
|
// access a snapshot of the memoryIndex (single-writer-principle)
|
||||||
|
HashMap memoryIndex = memoryREF.get(this);
|
||||||
|
return memoryIndex.size();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -212,8 +229,9 @@ class StorageBase {
|
|||||||
boolean contains(StorageKey key) {
|
boolean contains(StorageKey key) {
|
||||||
// protected by lock
|
// protected by lock
|
||||||
|
|
||||||
// check to see if it's in the pending ops
|
// access a snapshot of the memoryIndex (single-writer-principle)
|
||||||
return this.memoryIndex.containsKey(key);
|
HashMap memoryIndex = memoryREF.get(this);
|
||||||
|
return memoryIndex.containsKey(key);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -223,7 +241,11 @@ class StorageBase {
|
|||||||
<T> T getCached(StorageKey key) {
|
<T> T getCached(StorageKey key) {
|
||||||
// protected by lock
|
// protected by lock
|
||||||
|
|
||||||
Metadata meta = this.memoryIndex.get(key);
|
|
||||||
|
// access a snapshot of the memoryIndex (single-writer-principle)
|
||||||
|
HashMap memoryIndex = memoryREF.get(this);
|
||||||
|
Metadata meta = (Metadata) memoryIndex.get(key);
|
||||||
|
|
||||||
if (meta == null) {
|
if (meta == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@ -255,7 +277,9 @@ class StorageBase {
|
|||||||
<T> T get(StorageKey key) {
|
<T> T get(StorageKey key) {
|
||||||
// NOT protected by lock
|
// NOT protected by lock
|
||||||
|
|
||||||
Metadata meta = this.memoryIndex.get(key);
|
// access a snapshot of the memoryIndex (single-writer-principle)
|
||||||
|
HashMap memoryIndex = memoryREF.get(this);
|
||||||
|
Metadata meta = (Metadata) memoryIndex.get(key);
|
||||||
if (meta == null) {
|
if (meta == null) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@ -318,20 +342,37 @@ class StorageBase {
|
|||||||
final
|
final
|
||||||
boolean delete(StorageKey key) {
|
boolean delete(StorageKey key) {
|
||||||
// pending ops flushed (protected by lock)
|
// pending ops flushed (protected by lock)
|
||||||
// not protected by lock
|
|
||||||
Metadata delRec = this.memoryIndex.get(key);
|
|
||||||
|
|
||||||
try {
|
// synchronized is used here to ensure the "single writer principle", and make sure that ONLY one thread at a time can enter this
|
||||||
deleteRecordData(delRec, delRec.dataCapacity);
|
// section. Because of this, we can have unlimited reader threads all going at the same time, without contention.
|
||||||
deleteRecordIndex(key, delRec);
|
synchronized (singleWriterLock) {
|
||||||
return true;
|
Metadata delRec = this.memoryIndex.get(key);
|
||||||
} catch (IOException e) {
|
|
||||||
if (this.logger != null) {
|
try {
|
||||||
this.logger.error("Error while deleting data from disk", e);
|
deleteRecordData(delRec, delRec.dataCapacity);
|
||||||
} else {
|
|
||||||
e.printStackTrace();
|
// delete the record index
|
||||||
|
int currentNumRecords = this.memoryIndex.size();
|
||||||
|
if (delRec.indexPosition != currentNumRecords - 1) {
|
||||||
|
Metadata last = Metadata.readHeader(this.randomAccessFile, currentNumRecords - 1);
|
||||||
|
assert last != null;
|
||||||
|
|
||||||
|
last.moveRecord(this.randomAccessFile, delRec.indexPosition);
|
||||||
|
}
|
||||||
|
this.memoryIndex.remove(key);
|
||||||
|
|
||||||
|
|
||||||
|
setRecordCount(this.randomAccessFile, currentNumRecords - 1);
|
||||||
|
|
||||||
|
return true;
|
||||||
|
} catch (IOException e) {
|
||||||
|
if (this.logger != null) {
|
||||||
|
this.logger.error("Error while deleting data from disk", e);
|
||||||
|
} else {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -352,7 +393,12 @@ class StorageBase {
|
|||||||
.sync();
|
.sync();
|
||||||
this.input.close();
|
this.input.close();
|
||||||
this.randomAccessFile.close();
|
this.randomAccessFile.close();
|
||||||
this.memoryIndex.clear();
|
|
||||||
|
// synchronized is used here to ensure the "single writer principle", and make sure that ONLY one thread at a time can enter this
|
||||||
|
// section. Because of this, we can have unlimited reader threads all going at the same time, without contention.
|
||||||
|
synchronized (singleWriterLock) {
|
||||||
|
this.memoryIndex.clear();
|
||||||
|
}
|
||||||
|
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
if (this.logger != null) {
|
if (this.logger != null) {
|
||||||
@ -401,104 +447,110 @@ class StorageBase {
|
|||||||
*/
|
*/
|
||||||
private
|
private
|
||||||
void save0(StorageKey key, Object object) {
|
void save0(StorageKey key, Object object) {
|
||||||
Metadata metaData = this.memoryIndex.get(key);
|
Metadata metaData;
|
||||||
int currentRecordCount = this.numberOfRecords;
|
|
||||||
|
|
||||||
if (metaData != null) {
|
// synchronized is used here to ensure the "single writer principle", and make sure that ONLY one thread at a time can enter this
|
||||||
// now we have to UPDATE instead of add!
|
// section. Because of this, we can have unlimited reader threads all going at the same time, without contention.
|
||||||
try {
|
synchronized (singleWriterLock) {
|
||||||
if (currentRecordCount == 1) {
|
metaData = this.memoryIndex.get(key);
|
||||||
// if we are the ONLY one, then we can do things differently.
|
int currentRecordCount = this.numberOfRecords;
|
||||||
// just dump the data again to disk.
|
|
||||||
FileLock lock = this.randomAccessFile.getChannel()
|
|
||||||
.lock(this.dataPosition,
|
|
||||||
Long.MAX_VALUE - this.dataPosition,
|
|
||||||
false); // don't know how big it is, so max value it
|
|
||||||
|
|
||||||
this.randomAccessFile.seek(this.dataPosition); // this is the end of the file, we know this ahead-of-time
|
if (metaData != null) {
|
||||||
Metadata.writeData(this.serializationManager, object, this.output);
|
// now we have to UPDATE instead of add!
|
||||||
// have to re-specify the capacity and size
|
try {
|
||||||
//noinspection NumericCastThatLosesPrecision
|
if (currentRecordCount == 1) {
|
||||||
int sizeOfWrittenData = (int) (this.randomAccessFile.length() - this.dataPosition);
|
// if we are the ONLY one, then we can do things differently.
|
||||||
|
// just dump the data again to disk.
|
||||||
|
FileLock lock = this.randomAccessFile.getChannel()
|
||||||
|
.lock(this.dataPosition,
|
||||||
|
Long.MAX_VALUE - this.dataPosition,
|
||||||
|
false); // don't know how big it is, so max value it
|
||||||
|
|
||||||
metaData.dataCapacity = sizeOfWrittenData;
|
this.randomAccessFile.seek(this.dataPosition); // this is the end of the file, we know this ahead-of-time
|
||||||
metaData.dataCount = sizeOfWrittenData;
|
Metadata.writeData(this.serializationManager, object, this.output);
|
||||||
|
// have to re-specify the capacity and size
|
||||||
|
//noinspection NumericCastThatLosesPrecision
|
||||||
|
int sizeOfWrittenData = (int) (this.randomAccessFile.length() - this.dataPosition);
|
||||||
|
|
||||||
lock.release();
|
metaData.dataCapacity = sizeOfWrittenData;
|
||||||
}
|
metaData.dataCount = sizeOfWrittenData;
|
||||||
else {
|
|
||||||
// this is comparatively slow, since we serialize it first to get the size, then we put it in the file.
|
|
||||||
ByteArrayOutputStream dataStream = getDataAsByteArray(this.serializationManager, this.logger, object);
|
|
||||||
|
|
||||||
int size = dataStream.size();
|
lock.release();
|
||||||
if (size > metaData.dataCapacity) {
|
}
|
||||||
deleteRecordData(metaData, size);
|
else {
|
||||||
// stuff this record to the end of the file, since it won't fit in it's current location
|
// this is comparatively slow, since we serialize it first to get the size, then we put it in the file.
|
||||||
metaData.dataPointer = this.randomAccessFile.length();
|
ByteArrayOutputStream dataStream = getDataAsByteArray(this.serializationManager, this.logger, object);
|
||||||
// have to make sure that the CAPACITY of the new one is the SIZE of the new data!
|
|
||||||
// and since it is going to the END of the file, we do that.
|
int size = dataStream.size();
|
||||||
metaData.dataCapacity = size;
|
if (size > metaData.dataCapacity) {
|
||||||
metaData.dataCount = 0;
|
deleteRecordData(metaData, size);
|
||||||
|
// stuff this record to the end of the file, since it won't fit in it's current location
|
||||||
|
metaData.dataPointer = this.randomAccessFile.length();
|
||||||
|
// have to make sure that the CAPACITY of the new one is the SIZE of the new data!
|
||||||
|
// and since it is going to the END of the file, we do that.
|
||||||
|
metaData.dataCapacity = size;
|
||||||
|
metaData.dataCount = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: should check to see if the data is different. IF SO, then we write, otherwise nothing!
|
||||||
|
|
||||||
|
metaData.writeDataRaw(dataStream, this.randomAccessFile);
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: should check to see if the data is different. IF SO, then we write, otherwise nothing!
|
metaData.writeDataInfo(this.randomAccessFile);
|
||||||
|
} catch (IOException e) {
|
||||||
metaData.writeDataRaw(dataStream, this.randomAccessFile);
|
if (this.logger != null) {
|
||||||
}
|
this.logger.error("Error while saving data to disk", e);
|
||||||
|
} else {
|
||||||
metaData.writeDataInfo(this.randomAccessFile);
|
e.printStackTrace();
|
||||||
} catch (IOException e) {
|
}
|
||||||
if (this.logger != null) {
|
|
||||||
this.logger.error("Error while saving data to disk", e);
|
|
||||||
} else {
|
|
||||||
e.printStackTrace();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
else {
|
||||||
else {
|
// metadata == null...
|
||||||
// metadata == null...
|
try {
|
||||||
try {
|
// set the number of records that this storage has
|
||||||
// set the number of records that this storage has
|
setRecordCount(this.randomAccessFile, currentRecordCount + 1);
|
||||||
setRecordCount(this.randomAccessFile, currentRecordCount + 1);
|
|
||||||
|
|
||||||
// This will make sure that there is room to write a new record. This is zero indexed.
|
// This will make sure that there is room to write a new record. This is zero indexed.
|
||||||
// this will skip around if moves occur
|
// this will skip around if moves occur
|
||||||
ensureIndexCapacity(this.randomAccessFile);
|
ensureIndexCapacity(this.randomAccessFile);
|
||||||
|
|
||||||
// append record to end of file
|
// append record to end of file
|
||||||
long length = this.randomAccessFile.length();
|
long length = this.randomAccessFile.length();
|
||||||
|
|
||||||
// System.err.println("--Writing data to: " + length);
|
// System.err.println("--Writing data to: " + length);
|
||||||
|
|
||||||
metaData = new Metadata(key, currentRecordCount, length);
|
metaData = new Metadata(key, currentRecordCount, length);
|
||||||
metaData.writeMetaDataInfo(this.randomAccessFile);
|
metaData.writeMetaDataInfo(this.randomAccessFile);
|
||||||
|
|
||||||
// add new entry to the index
|
// add new entry to the index
|
||||||
this.memoryIndex.put(key, metaData);
|
this.memoryIndex.put(key, metaData);
|
||||||
|
|
||||||
// save out the data. Because we KNOW that we are writing this to the end of the file,
|
// save out the data. Because we KNOW that we are writing this to the end of the file,
|
||||||
// there are some tricks we can use.
|
// there are some tricks we can use.
|
||||||
|
|
||||||
// don't know how big it is, so max value it
|
// don't know how big it is, so max value it
|
||||||
FileLock lock = this.randomAccessFile.getChannel()
|
FileLock lock = this.randomAccessFile.getChannel()
|
||||||
.lock(0, Long.MAX_VALUE, false);
|
.lock(0, Long.MAX_VALUE, false);
|
||||||
|
|
||||||
// this is the end of the file, we know this ahead-of-time
|
// this is the end of the file, we know this ahead-of-time
|
||||||
this.randomAccessFile.seek(length);
|
this.randomAccessFile.seek(length);
|
||||||
|
|
||||||
int total = Metadata.writeData(this.serializationManager, object, this.output);
|
int total = Metadata.writeData(this.serializationManager, object, this.output);
|
||||||
lock.release();
|
lock.release();
|
||||||
|
|
||||||
metaData.dataCount = metaData.dataCapacity = total;
|
metaData.dataCount = metaData.dataCapacity = total;
|
||||||
// have to save it.
|
// have to save it.
|
||||||
metaData.writeDataInfo(this.randomAccessFile);
|
metaData.writeDataInfo(this.randomAccessFile);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
if (this.logger != null) {
|
if (this.logger != null) {
|
||||||
this.logger.error("Error while writing data to disk", e);
|
this.logger.error("Error while writing data to disk", e);
|
||||||
} else {
|
} else {
|
||||||
e.printStackTrace();
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -528,8 +580,8 @@ class StorageBase {
|
|||||||
// items to be "autosaved" are automatically injected into "actions".
|
// items to be "autosaved" are automatically injected into "actions".
|
||||||
final Set<Entry<StorageKey, Object>> entries = actions.entrySet();
|
final Set<Entry<StorageKey, Object>> entries = actions.entrySet();
|
||||||
for (Entry<StorageKey, Object> entry : entries) {
|
for (Entry<StorageKey, Object> entry : entries) {
|
||||||
Object object = entry.getValue();
|
|
||||||
StorageKey key = entry.getKey();
|
StorageKey key = entry.getKey();
|
||||||
|
Object object = entry.getValue();
|
||||||
|
|
||||||
// our action list is for explicitly saving objects (but not necessarily "registering" them to be auto-saved
|
// our action list is for explicitly saving objects (but not necessarily "registering" them to be auto-saved
|
||||||
save0(key, object);
|
save0(key, object);
|
||||||
@ -557,6 +609,7 @@ class StorageBase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// protected by singleWriterLock
|
||||||
private
|
private
|
||||||
void deleteRecordData(Metadata deletedRecord, int sizeOfDataToAdd) throws IOException {
|
void deleteRecordData(Metadata deletedRecord, int sizeOfDataToAdd) throws IOException {
|
||||||
if (this.randomAccessFile.length() == deletedRecord.dataPointer + deletedRecord.dataCapacity) {
|
if (this.randomAccessFile.length() == deletedRecord.dataPointer + deletedRecord.dataCapacity) {
|
||||||
@ -613,23 +666,6 @@ class StorageBase {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private
|
|
||||||
void deleteRecordIndex(StorageKey key, Metadata deleteRecord) throws IOException {
|
|
||||||
int currentNumRecords = this.memoryIndex.size();
|
|
||||||
|
|
||||||
if (deleteRecord.indexPosition != currentNumRecords - 1) {
|
|
||||||
Metadata last = Metadata.readHeader(this.randomAccessFile, currentNumRecords - 1);
|
|
||||||
assert last != null;
|
|
||||||
|
|
||||||
last.moveRecord(this.randomAccessFile, deleteRecord.indexPosition);
|
|
||||||
}
|
|
||||||
|
|
||||||
this.memoryIndex.remove(key);
|
|
||||||
|
|
||||||
setRecordCount(this.randomAccessFile, currentNumRecords - 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Writes the number of records header to the file.
|
* Writes the number of records header to the file.
|
||||||
*/
|
*/
|
||||||
@ -705,12 +741,13 @@ class StorageBase {
|
|||||||
*/
|
*/
|
||||||
private
|
private
|
||||||
Metadata index_getMetaDataFromData(long targetFp) {
|
Metadata index_getMetaDataFromData(long targetFp) {
|
||||||
Iterator<Metadata> iterator = this.memoryIndex.values()
|
// access a snapshot of the memoryIndex (single-writer-principle)
|
||||||
.iterator();
|
HashMap memoryIndex = memoryREF.get(this);
|
||||||
|
Iterator iterator = memoryIndex.values().iterator();
|
||||||
|
|
||||||
//noinspection WhileLoopReplaceableByForEach
|
//noinspection WhileLoopReplaceableByForEach
|
||||||
while (iterator.hasNext()) {
|
while (iterator.hasNext()) {
|
||||||
Metadata next = iterator.next();
|
Metadata next = (Metadata) iterator.next();
|
||||||
if (targetFp >= next.dataPointer && targetFp < next.dataPointer + next.dataCapacity) {
|
if (targetFp >= next.dataPointer && targetFp < next.dataPointer + next.dataCapacity) {
|
||||||
return next;
|
return next;
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user