dorkbox/Utilities
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Cleaned up storage. No more registered objects - it was a bad idea

Browse Source
This commit is contained in:
nathan 2014-09-16 15:39:33 +02:00
parent 7e9d9ac7b2
commit 957f2bfe4d
6 changed files with 202 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
Dorkbox-Util/src/dorkbox/util/FileUtil.java
View File

@ -618,38 +618,157 @@ public class FileUtil {
/**
* Gets the relative path of a file to a specific directory in it's hierarchy.
*
* For example: getRelativeToDir("/a/b/c/d/e.bah", "c") -> "d/e.bah"
* For example: getChildRelativeToDir("/a/b/c/d/e.bah", "c") -> "d/e.bah"
*/
public static String getRelativeToDir(String fileName, String dirInHeirarchy) {
public static String getChildRelativeToDir(String fileName, String dirInHeirarchy) {
if (fileName == null || fileName.isEmpty()) {
throw new IllegalArgumentException("fileName cannot be null.");
}
return getRelativeToDir(new File(fileName), dirInHeirarchy);
return getChildRelativeToDir(new File(fileName), dirInHeirarchy);
}
/**
* Gets the relative path of a file to a specific directory in it's hierarchy.
*
* For example: getRelativeToDir("/a/b/c/d/e.bah", "c") -> "d/e.bah"
* For example: getChildRelativeToDir("/a/b/c/d/e.bah", "c") -> "d/e.bah"
* @return null if it cannot be found
*/
public static String getRelativeToDir(File file, String dirInHeirarchy) {
public static String getChildRelativeToDir(File file, String dirInHeirarchy) {
if (file == null) {
throw new IllegalArgumentException("file cannot be null.");
}
if (dirInHeirarchy == null || dirInHeirarchy.isEmpty()) {
throw new IllegalArgumentException("dirInHeirarchy cannot be null.");
}
String[] split = dirInHeirarchy.split(File.separator);
int splitIndex = split.length-1;
String absolutePath = file.getAbsolutePath();
File parent = file;
String parentName;
while ((parent = parent.getParentFile()) != null) {
parentName = parent.getName();
if (parentName.equals(dirInHeirarchy)) {
parentName = parent.getAbsolutePath();
if (splitIndex == 0) {
// match on ONE dir
while (parent != null) {
parentName = parent.getName();
return absolutePath.substring(parentName.length() + 1);
if (parentName.equals(dirInHeirarchy)) {
parentName = parent.getAbsolutePath();
return absolutePath.substring(parentName.length() + 1);
}
parent = parent.getParentFile();
}
}
else {
// match on MANY dir. They must be "in-order"
boolean matched = false;
while (parent != null) {
parentName = parent.getName();
if (matched) {
if (parentName.equals(split[splitIndex])) {
splitIndex--;
if (splitIndex < 0) {
parent = parent.getParentFile();
parentName = parent.getAbsolutePath();
return absolutePath.substring(parentName.length() + 1);
}
} else {
// because it has to be "in-order", if it doesn't match, we immediately abort
return null;
}
} else {
if (parentName.equals(split[splitIndex])) {
matched = true;
splitIndex--;
}
}
parent = parent.getParentFile();
}
}
return null;
}
/**
* Gets the PARENT relative path of a file to a specific directory in it's hierarchy.
*
* For example: getParentRelativeToDir("/a/b/c/d/e.bah", "c") -> "/a/b"
*/
public static String getParentRelativeToDir(String fileName, String dirInHeirarchy) {
if (fileName == null || fileName.isEmpty()) {
throw new IllegalArgumentException("fileName cannot be null.");
}
return getParentRelativeToDir(new File(fileName), dirInHeirarchy);
}
/**
* Gets the relative path of a file to a specific directory in it's hierarchy.
*
* For example: getParentRelativeToDir("/a/b/c/d/e.bah", "c") -> "/a/b"
* @return null if it cannot be found
*/
public static String getParentRelativeToDir(File file, String dirInHeirarchy) {
if (file == null) {
throw new IllegalArgumentException("file cannot be null.");
}
if (dirInHeirarchy == null || dirInHeirarchy.isEmpty()) {
throw new IllegalArgumentException("dirInHeirarchy cannot be null.");
}
String[] split = dirInHeirarchy.split(File.separator);
int splitIndex = split.length-1;
File parent = file;
String parentName;
if (splitIndex == 0) {
// match on ONE dir
while (parent != null) {
parentName = parent.getName();
if (parentName.equals(dirInHeirarchy)) {
parent = parent.getParentFile();
parentName = parent.getAbsolutePath();
return parentName;
}
parent = parent.getParentFile();
}
}
else {
// match on MANY dir. They must be "in-order"
boolean matched = false;
while (parent != null) {
parentName = parent.getName();
if (matched) {
if (parentName.equals(split[splitIndex])) {
splitIndex--;
if (splitIndex < 0) {
parent = parent.getParentFile();
parentName = parent.getAbsolutePath();
return parentName;
}
} else {
// because it has to be "in-order", if it doesn't match, we immediately abort
return null;
}
} else {
if (parentName.equals(split[splitIndex])) {
matched = true;
splitIndex--;
}
}
parent = parent.getParentFile();
}
}

49
Dorkbox-Util/src/dorkbox/util/Sys.java
View File

@ -566,10 +566,13 @@ public class Sys {
// find ALL ServerLoader classes and use reflection to load them.
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
String noDotsPackageName = packageName;
boolean isEmpty = true;
if (packageName != null && !packageName.isEmpty()) {
packageName = packageName.replace('.', '/');
noDotsPackageName = packageName.replace('.', '/');
isEmpty = false;
} else {
packageName = ""; // cannot be null!
noDotsPackageName = ""; // cannot be null!
}
// look for all annotated classes in the projects package.
@ -577,16 +580,23 @@ public class Sys {
LinkedList<Class<?>> annotatedClasses = new LinkedList<Class<?>>();
URL url;
Enumeration<URL> resources = classLoader.getResources(packageName);
Enumeration<URL> resources = classLoader.getResources(noDotsPackageName);
// lengthy, but it will traverse how we want.
while (resources.hasMoreElements()) {
url = resources.nextElement();
if (url.getProtocol().equals("file")) {
File file = new File(url.getFile());
findAnnotatedClassesRecursive(classLoader, packageName, annotation, file, file.getAbsolutePath(), annotatedClasses);
if (!isEmpty) {
String relativeToDir = FileUtil.getParentRelativeToDir(file, noDotsPackageName);
if (relativeToDir != null) {
findAnnotatedClassesRecursive(classLoader, noDotsPackageName, annotation, file, relativeToDir, annotatedClasses);
}
} else {
findAnnotatedClassesRecursive(classLoader, noDotsPackageName, annotation, file, file.getAbsolutePath(), annotatedClasses);
}
} else {
findModulesInJar(classLoader, packageName, annotation, url, annotatedClasses);
findModulesInJar(classLoader, noDotsPackageName, annotation, url, annotatedClasses);
}
}
@ -617,7 +627,7 @@ public class Sys {
else if (isValid(fileName)) {
String classPath = absolutePath.substring(rootPath.length() + 1, absolutePath.length() - 6);
if (packageName.isEmpty()) {
if (!packageName.isEmpty()) {
if (!classPath.startsWith(packageName)) {
return;
}
@ -707,16 +717,25 @@ public class Sys {
}
int length = name.length();
boolean isValid = length > 6 &&
name.charAt(length-1) != '/' && // remove directories from the search.
name.charAt(length-6) == '.' &&
name.charAt(length-5) == 'c' &&
name.charAt(length-4) == 'l' &&
name.charAt(length-3) == 'a' &&
name.charAt(length-2) == 's' &&
name.charAt(length-1) == 's'; // make sure it's a class file
return isValid;
if (name.charAt(length-1) == '/') { // remove directories from the search.)
return false;
}
// ALSO, cannot use classes such as "ServerBloah$4.class".
int newLength = length-6;
for (int i=0;i<newLength;i++) {
if (name.charAt(i) == '$') {
return false;
}
}
return name.charAt(length-6) == '.' &&
name.charAt(length-5) == 'c' &&
name.charAt(length-4) == 'l' &&
name.charAt(length-3) == 'a' &&
name.charAt(length-2) == 's' &&
name.charAt(length-1) == 's'; // make sure it's a class file
}
/**

2
Dorkbox-Util/src/dorkbox/util/storage/Metadata.java
View File

@ -240,7 +240,7 @@ public class Metadata {
/**
* Writes data to the end of the file (which is where the datapointer is at)
*/
void writeDataToEndOfFile(Kryo kryo, Object data, DeflaterOutputStream outputStream) throws IOException {
void writeDataFast(Kryo kryo, Object data, DeflaterOutputStream outputStream) throws IOException {
Output output = new Output(outputStream, 1024); // write 1024 at a time
kryo.writeClassAndObject(output, data);
output.flush();

113
Dorkbox-Util/src/dorkbox/util/storage/Storage.java
View File

@ -7,7 +7,6 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
@ -222,6 +221,8 @@ public class Storage {
synchronized(storages) {
Collection<Storage> values = storages.values();
for (Storage storage : values) {
while (!storage.decrementReference()) {
}
storage.close();
}
storages.clear();
@ -262,8 +263,6 @@ public class Storage {
private final ReentrantLock actionLock = new ReentrantLock();
private volatile Map<ByteArrayWrapper, Object> actionMap = new ConcurrentHashMap<ByteArrayWrapper, Object>();
private Map<ByteArrayWrapper, Object> cacheMap = new HashMap<ByteArrayWrapper, Object>();
private AtomicBoolean isOpen = new AtomicBoolean(false);
@ -291,28 +290,7 @@ public class Storage {
actionLock2.unlock();
}
// anything in cacheMap must also be resaved, BUT ONLY IF IT's DIFFERENT!
Map<ByteArrayWrapper, Object> cacheMap2 = Storage.this.cacheMap;
StorageBase storage2 = Storage.this.storage;
synchronized (cacheMap2) {
if (!cacheMap2.isEmpty()) {
for (Entry<ByteArrayWrapper, Object> entry : cacheMap2.entrySet()) {
ByteArrayWrapper key = entry.getKey();
Object value = entry.getValue();
if (value != null && key != null) {
Object originalVersion = storage2.getCached(key);
if (!value.equals(originalVersion)) {
actions.put(key, value);
}
}
}
}
}
storage2.doActionThings(actions);
Storage.this.storage.doActionThings(actions);
}
});
@ -347,32 +325,6 @@ public class Storage {
return this.storage.contains(wrap(key));
}
/**
* Registers this key/value (object) pair to be automatically saved in a save operation
*
* @param key the key to save/register this object under
*/
public final void register(String key, Object object) {
ByteArrayWrapper wrap = wrap(key);
synchronized (this.cacheMap) {
this.cacheMap.put(wrap, object);
}
}
/**
* UN-Registers this key/value (object) pair, so it will no longer be automatically saved in a save operation.
*
* @param key the key to save/register this object under
*/
public final void unregister(String key) {
ByteArrayWrapper wrap = wrap(key);
synchronized (this.cacheMap) {
this.cacheMap.remove(wrap);
}
}
/**
* Reads a object using the default (blank) key
*/
@ -475,13 +427,6 @@ public class Storage {
ByteArrayWrapper wrap = wrap(key);
action(wrap, object);
synchronized (this.cacheMap) {
// if our cache has this key, update it!
if (this.cacheMap.containsKey(wrap)) {
this.cacheMap.put(wrap, object);
}
}
// timer action runs on TIMER thread, not this thread
this.timer.delay(this.milliSeconds);
}
@ -500,61 +445,12 @@ public class Storage {
ByteArrayWrapper wrap = wrap(key);
action(wrap, object);
synchronized (this.cacheMap) {
// if our cache has this key, update it!
if (this.cacheMap.containsKey(wrap)) {
this.cacheMap.put(wrap, object);
}
}
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
}
/**
* Saves all of the registered objects (and pending operations) to storage
* <p>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
public final void saveRegistered(String key) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
ByteArrayWrapper wrap = wrap(key);
synchronized (this.cacheMap) {
if (this.cacheMap.containsKey(wrap)) {
// timer action runs on TIMER thread, not this thread
this.timer.delay(this.milliSeconds);
}
}
}
/**
* Immediately saves all of the registered objects (and pending operations) to storage
* <p>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
*/
public final void saveRegisteredNow(String key) {
if (!this.isOpen.get()) {
throw new RuntimeException("Unable to act on closed storage");
}
ByteArrayWrapper wrap = wrap(key);
synchronized (this.cacheMap) {
if (this.cacheMap.containsKey(wrap)) {
// timer action runs on THIS thread, not timer thread
this.timer.delay(0L);
}
}
}
/**
* Adds the given object to the storage using a default (blank) key
* Adds the given object to the storage using a default (blank) key, OR -- if it has been registered, using it's registered key
* <p>
* Also will update existing data. If the new contents do not fit in the original space, then the update is handled by
* deleting the old data and adding the new.
@ -615,7 +511,6 @@ public class Storage {
this.timer.delay(0L);
this.storage.close();
this.cacheMap.clear();
}
/**

50
Dorkbox-Util/src/dorkbox/util/storage/StorageBase.java
View File

@ -327,20 +327,28 @@ public class StorageBase {
if (metaData != null) {
// now we have to UPDATE instead of add!
try {
ByteArrayOutputStream dataStream = metaData.getDataStream(this.kryo, object, deflater);
if (this.memoryIndex.size() == 1) {
// if we are the ONLY one, then we can do things differently.
// just dump the data agian to disk.
this.file.seek(this.dataPosition); // this is the end of the file, we know this ahead-of-time
metaData.writeDataFast(this.kryo, object, fileOutputStream);
} else {
// this is comparatively slow, since we serialize it first to get the size, then we put it in the file.
ByteArrayOutputStream dataStream = metaData.getDataStream(this.kryo, object, deflater);
int size = dataStream.size();
if (size > metaData.dataCapacity) {
deleteRecordData(metaData);
// stuff this record to the end of the file, since it won't fit in it's current location
metaData.dataPointer = this.file.length();
// have to make sure that the CAPACITY of the new one is the SIZE of the new data!
// and since it is going to the END of the file, we do that.
metaData.dataCapacity = size;
metaData.dataCount = 0;
int size = dataStream.size();
if (size > metaData.dataCapacity) {
deleteRecordData(metaData);
// stuff this record to the end of the file, since it won't fit in it's current location
metaData.dataPointer = this.file.length();
// have to make sure that the CAPACITY of the new one is the SIZE of the new data!
// and since it is going to the END of the file, we do that.
metaData.dataCapacity = size;
metaData.dataCount = 0;
}
metaData.writeData(dataStream, this.file);
}
metaData.writeData(dataStream, this.file);
metaData.writeDataInfo(this.file);
} catch (IOException e) {
this.logger.error("Error while saving data to disk", e);
@ -368,7 +376,7 @@ public class StorageBase {
// there are some tricks we can use.
this.file.seek(length); // this is the end of the file, we know this ahead-of-time
metaData.writeDataToEndOfFile(this.kryo, object, fileOutputStream);
metaData.writeDataFast(this.kryo, object, fileOutputStream);
metaData.dataCount = deflater.getTotalOut();
metaData.dataCapacity = metaData.dataCount;
@ -418,11 +426,19 @@ public class StorageBase {
previous.dataCapacity += deletedRecord.dataCapacity;
previous.writeDataInfo(this.file);
} else {
// the record to delete is the FIRST (of many) in the file.
// the FASTEST way to delete is to grow the records!
// Another option is to move the #2 data to the first data, but then there is the same gap after #2.
Metadata secondRecord = index_getMetaDataFromData(deletedRecord.dataPointer + deletedRecord.dataCapacity + 1);
setDataPosition(this.file, secondRecord.dataPointer);
// because there is no "previous", that means we MIGHT be the FIRST record
Metadata first = index_getMetaDataFromData(this.dataPosition);
if (first == deletedRecord) {
// the record to delete is the FIRST (of many) in the file.
// the FASTEST way to delete is to grow the number of allowed records!
// Another option is to move the #2 data to the first data, but then there is the same gap after #2.
setDataPosition(this.file, deletedRecord.dataPointer + deletedRecord.dataCapacity);
} else {
// well, we're not the first record. which one is RIGHT before us?
// it should be "previous", so something fucked up
this.logger.error("Trying to delete an object, and it's in a weird state");
}
}
}

2
Dorkbox-Util/test/dorkbox/util/StorageTest.java
View File

@ -366,7 +366,7 @@ public class StorageTest {
makeData(data);
String createKey = createKey(i);
storage.register(createKey, data);
storage.save(createKey, data);
}
Storage.close(storage);