Cleaned up storage. No more registered objects - it was a bad idea
This commit is contained in:
parent
7e9d9ac7b2
commit
957f2bfe4d
@ -618,38 +618,157 @@ public class FileUtil {
|
||||
/**
|
||||
* Gets the relative path of a file to a specific directory in it's hierarchy.
|
||||
*
|
||||
* For example: getRelativeToDir("/a/b/c/d/e.bah", "c") -> "d/e.bah"
|
||||
* For example: getChildRelativeToDir("/a/b/c/d/e.bah", "c") -> "d/e.bah"
|
||||
*/
|
||||
public static String getRelativeToDir(String fileName, String dirInHeirarchy) {
|
||||
public static String getChildRelativeToDir(String fileName, String dirInHeirarchy) {
|
||||
if (fileName == null || fileName.isEmpty()) {
|
||||
throw new IllegalArgumentException("fileName cannot be null.");
|
||||
}
|
||||
|
||||
return getRelativeToDir(new File(fileName), dirInHeirarchy);
|
||||
return getChildRelativeToDir(new File(fileName), dirInHeirarchy);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the relative path of a file to a specific directory in it's hierarchy.
|
||||
*
|
||||
* For example: getRelativeToDir("/a/b/c/d/e.bah", "c") -> "d/e.bah"
|
||||
* For example: getChildRelativeToDir("/a/b/c/d/e.bah", "c") -> "d/e.bah"
|
||||
* @return null if it cannot be found
|
||||
*/
|
||||
public static String getRelativeToDir(File file, String dirInHeirarchy) {
|
||||
public static String getChildRelativeToDir(File file, String dirInHeirarchy) {
|
||||
if (file == null) {
|
||||
throw new IllegalArgumentException("file cannot be null.");
|
||||
}
|
||||
|
||||
if (dirInHeirarchy == null || dirInHeirarchy.isEmpty()) {
|
||||
throw new IllegalArgumentException("dirInHeirarchy cannot be null.");
|
||||
}
|
||||
|
||||
String[] split = dirInHeirarchy.split(File.separator);
|
||||
int splitIndex = split.length-1;
|
||||
|
||||
String absolutePath = file.getAbsolutePath();
|
||||
|
||||
File parent = file;
|
||||
String parentName;
|
||||
while ((parent = parent.getParentFile()) != null) {
|
||||
parentName = parent.getName();
|
||||
|
||||
if (parentName.equals(dirInHeirarchy)) {
|
||||
parentName = parent.getAbsolutePath();
|
||||
if (splitIndex == 0) {
|
||||
// match on ONE dir
|
||||
while (parent != null) {
|
||||
parentName = parent.getName();
|
||||
|
||||
return absolutePath.substring(parentName.length() + 1);
|
||||
if (parentName.equals(dirInHeirarchy)) {
|
||||
parentName = parent.getAbsolutePath();
|
||||
|
||||
return absolutePath.substring(parentName.length() + 1);
|
||||
}
|
||||
parent = parent.getParentFile();
|
||||
}
|
||||
}
|
||||
else {
|
||||
// match on MANY dir. They must be "in-order"
|
||||
boolean matched = false;
|
||||
while (parent != null) {
|
||||
parentName = parent.getName();
|
||||
|
||||
if (matched) {
|
||||
if (parentName.equals(split[splitIndex])) {
|
||||
splitIndex--;
|
||||
if (splitIndex < 0) {
|
||||
parent = parent.getParentFile();
|
||||
parentName = parent.getAbsolutePath();
|
||||
return absolutePath.substring(parentName.length() + 1);
|
||||
}
|
||||
} else {
|
||||
// because it has to be "in-order", if it doesn't match, we immediately abort
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
if (parentName.equals(split[splitIndex])) {
|
||||
matched = true;
|
||||
splitIndex--;
|
||||
}
|
||||
}
|
||||
|
||||
parent = parent.getParentFile();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the PARENT relative path of a file to a specific directory in it's hierarchy.
|
||||
*
|
||||
* For example: getParentRelativeToDir("/a/b/c/d/e.bah", "c") -> "/a/b"
|
||||
*/
|
||||
public static String getParentRelativeToDir(String fileName, String dirInHeirarchy) {
|
||||
if (fileName == null || fileName.isEmpty()) {
|
||||
throw new IllegalArgumentException("fileName cannot be null.");
|
||||
}
|
||||
|
||||
return getParentRelativeToDir(new File(fileName), dirInHeirarchy);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the relative path of a file to a specific directory in it's hierarchy.
|
||||
*
|
||||
* For example: getParentRelativeToDir("/a/b/c/d/e.bah", "c") -> "/a/b"
|
||||
* @return null if it cannot be found
|
||||
*/
|
||||
public static String getParentRelativeToDir(File file, String dirInHeirarchy) {
|
||||
if (file == null) {
|
||||
throw new IllegalArgumentException("file cannot be null.");
|
||||
}
|
||||
|
||||
if (dirInHeirarchy == null || dirInHeirarchy.isEmpty()) {
|
||||
throw new IllegalArgumentException("dirInHeirarchy cannot be null.");
|
||||
}
|
||||
|
||||
String[] split = dirInHeirarchy.split(File.separator);
|
||||
int splitIndex = split.length-1;
|
||||
|
||||
File parent = file;
|
||||
String parentName;
|
||||
|
||||
if (splitIndex == 0) {
|
||||
// match on ONE dir
|
||||
while (parent != null) {
|
||||
parentName = parent.getName();
|
||||
|
||||
if (parentName.equals(dirInHeirarchy)) {
|
||||
parent = parent.getParentFile();
|
||||
parentName = parent.getAbsolutePath();
|
||||
return parentName;
|
||||
}
|
||||
parent = parent.getParentFile();
|
||||
}
|
||||
}
|
||||
else {
|
||||
// match on MANY dir. They must be "in-order"
|
||||
boolean matched = false;
|
||||
while (parent != null) {
|
||||
parentName = parent.getName();
|
||||
|
||||
if (matched) {
|
||||
if (parentName.equals(split[splitIndex])) {
|
||||
splitIndex--;
|
||||
if (splitIndex < 0) {
|
||||
parent = parent.getParentFile();
|
||||
parentName = parent.getAbsolutePath();
|
||||
return parentName;
|
||||
}
|
||||
} else {
|
||||
// because it has to be "in-order", if it doesn't match, we immediately abort
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
if (parentName.equals(split[splitIndex])) {
|
||||
matched = true;
|
||||
splitIndex--;
|
||||
}
|
||||
}
|
||||
|
||||
parent = parent.getParentFile();
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -566,10 +566,13 @@ public class Sys {
|
||||
// find ALL ServerLoader classes and use reflection to load them.
|
||||
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
|
||||
|
||||
String noDotsPackageName = packageName;
|
||||
boolean isEmpty = true;
|
||||
if (packageName != null && !packageName.isEmpty()) {
|
||||
packageName = packageName.replace('.', '/');
|
||||
noDotsPackageName = packageName.replace('.', '/');
|
||||
isEmpty = false;
|
||||
} else {
|
||||
packageName = ""; // cannot be null!
|
||||
noDotsPackageName = ""; // cannot be null!
|
||||
}
|
||||
|
||||
// look for all annotated classes in the projects package.
|
||||
@ -577,16 +580,23 @@ public class Sys {
|
||||
LinkedList<Class<?>> annotatedClasses = new LinkedList<Class<?>>();
|
||||
|
||||
URL url;
|
||||
Enumeration<URL> resources = classLoader.getResources(packageName);
|
||||
Enumeration<URL> resources = classLoader.getResources(noDotsPackageName);
|
||||
|
||||
// lengthy, but it will traverse how we want.
|
||||
while (resources.hasMoreElements()) {
|
||||
url = resources.nextElement();
|
||||
if (url.getProtocol().equals("file")) {
|
||||
File file = new File(url.getFile());
|
||||
findAnnotatedClassesRecursive(classLoader, packageName, annotation, file, file.getAbsolutePath(), annotatedClasses);
|
||||
if (!isEmpty) {
|
||||
String relativeToDir = FileUtil.getParentRelativeToDir(file, noDotsPackageName);
|
||||
if (relativeToDir != null) {
|
||||
findAnnotatedClassesRecursive(classLoader, noDotsPackageName, annotation, file, relativeToDir, annotatedClasses);
|
||||
}
|
||||
} else {
|
||||
findAnnotatedClassesRecursive(classLoader, noDotsPackageName, annotation, file, file.getAbsolutePath(), annotatedClasses);
|
||||
}
|
||||
} else {
|
||||
findModulesInJar(classLoader, packageName, annotation, url, annotatedClasses);
|
||||
findModulesInJar(classLoader, noDotsPackageName, annotation, url, annotatedClasses);
|
||||
}
|
||||
}
|
||||
|
||||
@ -617,7 +627,7 @@ public class Sys {
|
||||
else if (isValid(fileName)) {
|
||||
String classPath = absolutePath.substring(rootPath.length() + 1, absolutePath.length() - 6);
|
||||
|
||||
if (packageName.isEmpty()) {
|
||||
if (!packageName.isEmpty()) {
|
||||
if (!classPath.startsWith(packageName)) {
|
||||
return;
|
||||
}
|
||||
@ -707,16 +717,25 @@ public class Sys {
|
||||
}
|
||||
|
||||
int length = name.length();
|
||||
boolean isValid = length > 6 &&
|
||||
name.charAt(length-1) != '/' && // remove directories from the search.
|
||||
name.charAt(length-6) == '.' &&
|
||||
name.charAt(length-5) == 'c' &&
|
||||
name.charAt(length-4) == 'l' &&
|
||||
name.charAt(length-3) == 'a' &&
|
||||
name.charAt(length-2) == 's' &&
|
||||
name.charAt(length-1) == 's'; // make sure it's a class file
|
||||
|
||||
return isValid;
|
||||
if (name.charAt(length-1) == '/') { // remove directories from the search.)
|
||||
return false;
|
||||
}
|
||||
|
||||
// ALSO, cannot use classes such as "ServerBloah$4.class".
|
||||
int newLength = length-6;
|
||||
for (int i=0;i<newLength;i++) {
|
||||
if (name.charAt(i) == '$') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return name.charAt(length-6) == '.' &&
|
||||
name.charAt(length-5) == 'c' &&
|
||||
name.charAt(length-4) == 'l' &&
|
||||
name.charAt(length-3) == 'a' &&
|
||||
name.charAt(length-2) == 's' &&
|
||||
name.charAt(length-1) == 's'; // make sure it's a class file
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -240,7 +240,7 @@ public class Metadata {
|
||||
/**
|
||||
* Writes data to the end of the file (which is where the datapointer is at)
|
||||
*/
|
||||
void writeDataToEndOfFile(Kryo kryo, Object data, DeflaterOutputStream outputStream) throws IOException {
|
||||
void writeDataFast(Kryo kryo, Object data, DeflaterOutputStream outputStream) throws IOException {
|
||||
Output output = new Output(outputStream, 1024); // write 1024 at a time
|
||||
kryo.writeClassAndObject(output, data);
|
||||
output.flush();
|
||||
|
@ -7,7 +7,6 @@ import java.util.Collection;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
@ -222,6 +221,8 @@ public class Storage {
|
||||
synchronized(storages) {
|
||||
Collection<Storage> values = storages.values();
|
||||
for (Storage storage : values) {
|
||||
while (!storage.decrementReference()) {
|
||||
}
|
||||
storage.close();
|
||||
}
|
||||
storages.clear();
|
||||
@ -262,8 +263,6 @@ public class Storage {
|
||||
private final ReentrantLock actionLock = new ReentrantLock();
|
||||
private volatile Map<ByteArrayWrapper, Object> actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
|
||||
|
||||
private Map<ByteArrayWrapper, Object> cacheMap = new HashMap<ByteArrayWrapper, Object>();
|
||||
|
||||
private AtomicBoolean isOpen = new AtomicBoolean(false);
|
||||
|
||||
|
||||
@ -291,28 +290,7 @@ public class Storage {
|
||||
actionLock2.unlock();
|
||||
}
|
||||
|
||||
// anything in cacheMap must also be resaved, BUT ONLY IF IT's DIFFERENT!
|
||||
Map<ByteArrayWrapper, Object> cacheMap2 = Storage.this.cacheMap;
|
||||
StorageBase storage2 = Storage.this.storage;
|
||||
|
||||
synchronized (cacheMap2) {
|
||||
if (!cacheMap2.isEmpty()) {
|
||||
for (Entry<ByteArrayWrapper, Object> entry : cacheMap2.entrySet()) {
|
||||
ByteArrayWrapper key = entry.getKey();
|
||||
Object value = entry.getValue();
|
||||
|
||||
if (value != null && key != null) {
|
||||
Object originalVersion = storage2.getCached(key);
|
||||
|
||||
if (!value.equals(originalVersion)) {
|
||||
actions.put(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
storage2.doActionThings(actions);
|
||||
Storage.this.storage.doActionThings(actions);
|
||||
}
|
||||
});
|
||||
|
||||
@ -347,32 +325,6 @@ public class Storage {
|
||||
return this.storage.contains(wrap(key));
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers this key/value (object) pair to be automatically saved in a save operation
|
||||
*
|
||||
* @param key the key to save/register this object under
|
||||
*/
|
||||
public final void register(String key, Object object) {
|
||||
ByteArrayWrapper wrap = wrap(key);
|
||||
|
||||
synchronized (this.cacheMap) {
|
||||
this.cacheMap.put(wrap, object);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* UN-Registers this key/value (object) pair, so it will no longer be automatically saved in a save operation.
|
||||
*
|
||||
* @param key the key to save/register this object under
|
||||
*/
|
||||
public final void unregister(String key) {
|
||||
ByteArrayWrapper wrap = wrap(key);
|
||||
|
||||
synchronized (this.cacheMap) {
|
||||
this.cacheMap.remove(wrap);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a object using the default (blank) key
|
||||
*/
|
||||
@ -475,13 +427,6 @@ public class Storage {
|
||||
ByteArrayWrapper wrap = wrap(key);
|
||||
action(wrap, object);
|
||||
|
||||
synchronized (this.cacheMap) {
|
||||
// if our cache has this key, update it!
|
||||
if (this.cacheMap.containsKey(wrap)) {
|
||||
this.cacheMap.put(wrap, object);
|
||||
}
|
||||
}
|
||||
|
||||
// timer action runs on TIMER thread, not this thread
|
||||
this.timer.delay(this.milliSeconds);
|
||||
}
|
||||
@ -500,61 +445,12 @@ public class Storage {
|
||||
ByteArrayWrapper wrap = wrap(key);
|
||||
action(wrap, object);
|
||||
|
||||
synchronized (this.cacheMap) {
|
||||
// if our cache has this key, update it!
|
||||
if (this.cacheMap.containsKey(wrap)) {
|
||||
this.cacheMap.put(wrap, object);
|
||||
}
|
||||
}
|
||||
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
}
|
||||
|
||||
/**
|
||||
* Saves all of the registered objects (and pending operations) to storage
|
||||
* <p>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
public final void saveRegistered(String key) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
ByteArrayWrapper wrap = wrap(key);
|
||||
|
||||
synchronized (this.cacheMap) {
|
||||
if (this.cacheMap.containsKey(wrap)) {
|
||||
// timer action runs on TIMER thread, not this thread
|
||||
this.timer.delay(this.milliSeconds);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Immediately saves all of the registered objects (and pending operations) to storage
|
||||
* <p>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
*/
|
||||
public final void saveRegisteredNow(String key) {
|
||||
if (!this.isOpen.get()) {
|
||||
throw new RuntimeException("Unable to act on closed storage");
|
||||
}
|
||||
|
||||
ByteArrayWrapper wrap = wrap(key);
|
||||
|
||||
synchronized (this.cacheMap) {
|
||||
if (this.cacheMap.containsKey(wrap)) {
|
||||
// timer action runs on THIS thread, not timer thread
|
||||
this.timer.delay(0L);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds the given object to the storage using a default (blank) key
|
||||
* Adds the given object to the storage using a default (blank) key, OR -- if it has been registered, using it's registered key
|
||||
* <p>
|
||||
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
|
||||
* deleting the old data and adding the new.
|
||||
@ -615,7 +511,6 @@ public class Storage {
|
||||
this.timer.delay(0L);
|
||||
|
||||
this.storage.close();
|
||||
this.cacheMap.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -327,20 +327,28 @@ public class StorageBase {
|
||||
if (metaData != null) {
|
||||
// now we have to UPDATE instead of add!
|
||||
try {
|
||||
ByteArrayOutputStream dataStream = metaData.getDataStream(this.kryo, object, deflater);
|
||||
if (this.memoryIndex.size() == 1) {
|
||||
// if we are the ONLY one, then we can do things differently.
|
||||
// just dump the data agian to disk.
|
||||
this.file.seek(this.dataPosition); // this is the end of the file, we know this ahead-of-time
|
||||
metaData.writeDataFast(this.kryo, object, fileOutputStream);
|
||||
} else {
|
||||
// this is comparatively slow, since we serialize it first to get the size, then we put it in the file.
|
||||
ByteArrayOutputStream dataStream = metaData.getDataStream(this.kryo, object, deflater);
|
||||
|
||||
int size = dataStream.size();
|
||||
if (size > metaData.dataCapacity) {
|
||||
deleteRecordData(metaData);
|
||||
// stuff this record to the end of the file, since it won't fit in it's current location
|
||||
metaData.dataPointer = this.file.length();
|
||||
// have to make sure that the CAPACITY of the new one is the SIZE of the new data!
|
||||
// and since it is going to the END of the file, we do that.
|
||||
metaData.dataCapacity = size;
|
||||
metaData.dataCount = 0;
|
||||
int size = dataStream.size();
|
||||
if (size > metaData.dataCapacity) {
|
||||
deleteRecordData(metaData);
|
||||
// stuff this record to the end of the file, since it won't fit in it's current location
|
||||
metaData.dataPointer = this.file.length();
|
||||
// have to make sure that the CAPACITY of the new one is the SIZE of the new data!
|
||||
// and since it is going to the END of the file, we do that.
|
||||
metaData.dataCapacity = size;
|
||||
metaData.dataCount = 0;
|
||||
}
|
||||
metaData.writeData(dataStream, this.file);
|
||||
}
|
||||
|
||||
metaData.writeData(dataStream, this.file);
|
||||
metaData.writeDataInfo(this.file);
|
||||
} catch (IOException e) {
|
||||
this.logger.error("Error while saving data to disk", e);
|
||||
@ -368,7 +376,7 @@ public class StorageBase {
|
||||
// there are some tricks we can use.
|
||||
this.file.seek(length); // this is the end of the file, we know this ahead-of-time
|
||||
|
||||
metaData.writeDataToEndOfFile(this.kryo, object, fileOutputStream);
|
||||
metaData.writeDataFast(this.kryo, object, fileOutputStream);
|
||||
|
||||
metaData.dataCount = deflater.getTotalOut();
|
||||
metaData.dataCapacity = metaData.dataCount;
|
||||
@ -418,11 +426,19 @@ public class StorageBase {
|
||||
previous.dataCapacity += deletedRecord.dataCapacity;
|
||||
previous.writeDataInfo(this.file);
|
||||
} else {
|
||||
// the record to delete is the FIRST (of many) in the file.
|
||||
// the FASTEST way to delete is to grow the records!
|
||||
// Another option is to move the #2 data to the first data, but then there is the same gap after #2.
|
||||
Metadata secondRecord = index_getMetaDataFromData(deletedRecord.dataPointer + deletedRecord.dataCapacity + 1);
|
||||
setDataPosition(this.file, secondRecord.dataPointer);
|
||||
// because there is no "previous", that means we MIGHT be the FIRST record
|
||||
Metadata first = index_getMetaDataFromData(this.dataPosition);
|
||||
|
||||
if (first == deletedRecord) {
|
||||
// the record to delete is the FIRST (of many) in the file.
|
||||
// the FASTEST way to delete is to grow the number of allowed records!
|
||||
// Another option is to move the #2 data to the first data, but then there is the same gap after #2.
|
||||
setDataPosition(this.file, deletedRecord.dataPointer + deletedRecord.dataCapacity);
|
||||
} else {
|
||||
// well, we're not the first record. which one is RIGHT before us?
|
||||
// it should be "previous", so something fucked up
|
||||
this.logger.error("Trying to delete an object, and it's in a weird state");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -366,7 +366,7 @@ public class StorageTest {
|
||||
makeData(data);
|
||||
String createKey = createKey(i);
|
||||
|
||||
storage.register(createKey, data);
|
||||
storage.save(createKey, data);
|
||||
}
|
||||
Storage.close(storage);
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user