dorkbox/Network
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Now use the storage project

Browse Source
This commit is contained in:
Robinson 2021-08-23 00:39:55 -06:00
parent 87338c6f24
commit a75ece2a39
13 changed files with 112 additions and 336 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/dorkbox/network/Configuration.kt
View File

@ -23,9 +23,8 @@ import dorkbox.network.aeron.CoroutineIdleStrategy
import dorkbox.network.aeron.CoroutineSleepingMillisIdleStrategy
import dorkbox.network.connection.Connection
import dorkbox.network.serialization.Serialization
import dorkbox.network.storage.StorageType
import dorkbox.network.storage.types.PropertyStore
import dorkbox.os.OS
import dorkbox.storage.Storage
import io.aeron.driver.Configuration
import io.aeron.driver.ThreadingMode
import io.aeron.exceptions.DriverTimeoutException
@ -92,16 +91,8 @@ class ServerConfiguration : dorkbox.network.Configuration() {
/**
* Allows the user to change how endpoint settings and public key information are saved.
*
* For example, a custom database instead of the default, in-memory storage.
*
* Included types are:
* * ChronicleMapStore.type(file) -- high performance, but non-transactional and not recommended to be shared
* * LmdbStore.type(file) -- high performance, ACID, and can be shared
* * MemoryStore.type() -- v. high performance, but not persistent
* * PropertyStore.type(file) -- slow performance on write, but can easily be edited by user (similar to how openSSH server key info is)
*/
override var settingsStore: StorageType = PropertyStore.type("settings-server.db")
override var settingsStore: Storage.Builder = Storage.Property().file("settings-server.db")
set(value) {
require(!contextDefined) { errorMessage }
field = value
@ -265,17 +256,9 @@ open class Configuration {
/**
* Allows the user to change how endpoint settings and public key information are saved.
*
* For example, a custom database instead of the default, in-memory storage.
*
* Included types are:
* * ChronicleMapStore.type(file) -- high performance, but non-transactional and not recommended to be shared
* * LmdbStore.type(file) -- high performance, ACID, and can be shared
* * MemoryStore.type() -- v. high performance, but not persistent
* * PropertyStore.type(file) -- slow performance on write, but can easily be edited by user (similar to how openSSH server key info is)
*
* Note: This field is overridden for server configurations, so that the file used is different for client/server
*/
open var settingsStore: StorageType = PropertyStore.type("settings-client.db")
open var settingsStore: Storage.Builder = Storage.Property().file("settings-client.db")
set(value) {
require(!contextDefined) { errorMessage }
field = value

30
src/dorkbox/network/storage/GenericStore.kt
View File

@ -1,30 +0,0 @@
/*
* Copyright 2020 dorkbox, llc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package dorkbox.network.storage
interface GenericStore: AutoCloseable {
/**
* similar to map.get()
*/
operator fun get(key: Any): ByteArray?
/**
* similar to map.set()
*
* Setting to NULL removes the value
*/
operator fun set(key: Any, bytes: ByteArray?)
}

85
src/dorkbox/network/storage/SettingsStore.kt
View File

@ -15,10 +15,17 @@
*/
package dorkbox.network.storage
import dorkbox.bytes.decodeBase58
import dorkbox.bytes.encodeToBase58String
import dorkbox.netUtil.IP
import dorkbox.netUtil.IPv4
import dorkbox.netUtil.IPv6
import dorkbox.network.connection.CryptoManagement
import dorkbox.serializers.SerializationDefaults
import dorkbox.storage.Storage
import mu.KLogger
import java.net.Inet4Address
import java.net.Inet6Address
import java.net.InetAddress
import java.security.SecureRandom
@ -26,7 +33,7 @@ import java.security.SecureRandom
* This class provides a way for the network stack to use a database of some sort.
*/
@Suppress("unused")
class SettingsStore(val logger: KLogger, val store: GenericStore) : AutoCloseable {
class SettingsStore(storageBuilder: Storage.Builder, val logger: KLogger) : AutoCloseable {
companion object {
/**
* Address 0.0.0.0 or ::0 may be used as a source address for this host on this network.
@ -40,9 +47,83 @@ class SettingsStore(val logger: KLogger, val store: GenericStore) : AutoCloseabl
internal const val privateKey = "_private"
}
val store: Storage
init {
store = storageBuilder.logger(logger).apply {
if (!isStringBased) {
// have to load/save keys+values as strings
onLoad { key, value, load ->
// key/value will be strings for a string based storage system
key as String
value as String
// we want the keys to be easy to read in case we are using string based storage
val xKey: Any? = when (key) {
saltKey, privateKey -> key
else -> {
IP.toAddress(key)
}
}
if (xKey == null) {
logger.error("Unable to parse onLoad key property [$key] $value")
return@onLoad
}
val xValue = value.decodeBase58()
load(xKey, xValue)
}.onSave { key, value, save ->
// we want the keys to be easy to read in case we are using string based storage
val xKey = when (key) {
saltKey, privateKey, Storage.versionTag -> key
is InetAddress -> IP.toString(key)
else -> null
}
if (xKey == null) {
logger.error("Unable to parse onSave key property [$key] $value")
return@onSave
}
val xValue = when(value) {
is Long -> value.toString()
is ByteArray -> value.encodeToBase58String()
else -> null
}
if (xValue == null) {
logger.error("Unable to parse onSave value property [$key] $value")
return@onSave
}
// all values are stored as bytes
save(xKey, xValue)
}
} else {
// everything is stored as bytes. We use a serializer instead to register types for easy serialization
onNewSerializer {
register(ByteArray::class.java)
register(Inet4Address::class.java, SerializationDefaults.inet4AddressSerializer)
register(Inet6Address::class.java, SerializationDefaults.inet6AddressSerializer)
}
}
}.build()
// have to init salt
if (store[saltKey] == null) {
val currentValue: ByteArray? = store[saltKey]
if (currentValue == null) {
val secureRandom = SecureRandom()
// server salt is used to salt usernames and other various connection handshake parameters

26
src/dorkbox/network/storage/StorageType.kt
View File

@ -1,26 +0,0 @@
/*
* Copyright 2020 dorkbox, llc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package dorkbox.network.storage
import mu.KLogger
import mu.KotlinLogging
interface StorageType {
/**
* Creates the custom storage type
*/
fun create(logger: KLogger = KotlinLogging.logger("StorageType")): SettingsStore
}

54
src/dorkbox/network/storage/types/MemoryStore.kt
View File

@ -1,54 +0,0 @@
/*
* Copyright 2020 dorkbox, llc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package dorkbox.network.storage.types
import dorkbox.network.storage.GenericStore
import dorkbox.network.storage.SettingsStore
import dorkbox.network.storage.StorageType
import mu.KLogger
import org.agrona.collections.Object2ObjectHashMap
/**
* In-Memory store
*/
object MemoryStore {
fun type() = object : StorageType {
override fun create(logger: KLogger): SettingsStore {
return SettingsStore(logger, MemoryAccess(logger))
}
}
}
class MemoryAccess(val logger: KLogger): GenericStore {
private val map = Object2ObjectHashMap<Any, ByteArray>()
init {
logger.info("Memory storage initialized")
}
override fun get(key: Any): ByteArray? {
return map[key]
}
override fun set(key: Any, bytes: ByteArray?) {
map[key] = bytes
}
override fun close() {
}
}

169
src/dorkbox/network/storage/types/PropertyStore.kt
View File

@ -1,169 +0,0 @@
/*
* Copyright 2020 dorkbox, llc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package dorkbox.network.storage.types
import dorkbox.netUtil.IP
import dorkbox.network.storage.GenericStore
import dorkbox.network.storage.SettingsStore
import dorkbox.network.storage.StorageType
import dorkbox.util.Sys
import dorkbox.util.properties.SortedProperties
import mu.KLogger
import org.agrona.collections.Object2ObjectHashMap
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.io.IOException
import java.net.InetAddress
import java.util.*
/**
* Java property files
*/
class PropertyStore(val dbFile: File, val logger: KLogger): GenericStore {
companion object {
fun type(dbFile: String) : StorageType {
return type(File(dbFile))
}
fun type(dbFile: File) = object : StorageType {
override fun create(logger: KLogger): SettingsStore {
return SettingsStore(logger, PropertyStore (dbFile, logger))
}
}
}
@Volatile
private var lastModifiedTime = 0L
private val loadedProps = Object2ObjectHashMap<Any, ByteArray>()
init {
load()
logger.info("Property file storage initialized at: '$dbFile'")
}
private fun load() {
// if we cannot load, then we create a properties file.
if (!dbFile.canRead() && !dbFile.createNewFile()) {
throw IOException("Cannot create file")
}
val input = FileInputStream(dbFile)
try {
val properties = Properties()
properties.load(input)
lastModifiedTime = dbFile.lastModified()
properties.entries.forEach {
val key = it.key as String
val value = it.value as String
when (key) {
SettingsStore.saltKey -> loadedProps[SettingsStore.saltKey] = Sys.hexToBytes(value)
SettingsStore.privateKey -> loadedProps[SettingsStore.privateKey] = Sys.hexToBytes(value)
else -> {
val address: InetAddress? = IP.toAddress(key)
if (address != null) {
loadedProps[address] = Sys.hexToBytes(value)
} else {
logger.error("Unable to parse property file: $dbFile $key $value")
}
}
}
}
properties.clear()
} catch (e: IOException) {
logger.error("Cannot load properties!", e)
e.printStackTrace()
} finally {
input.close()
}
}
override operator fun get(key: Any): ByteArray? {
// we want to check the last modified time when getting, because if we edit the on-disk file, we want to those changes
val lastModifiedTime = dbFile.lastModified()
if (this.lastModifiedTime != lastModifiedTime) {
// we want to reload the info
load()
}
val any = loadedProps[key]
if (any != null) {
return any
}
return null
}
/**
* Setting to NULL removes it
*/
override operator fun set(key: Any, bytes: ByteArray?) {
val hasChanged = if (bytes == null) {
loadedProps.remove(key) != null
} else {
val prev = loadedProps.put(key, bytes)
!prev.contentEquals(bytes)
}
// every time we set info, we want to save it to disk (so the file on disk will ALWAYS be current, and so we can modify it as we choose)
if (hasChanged) {
save()
}
}
fun save() {
var fos: FileOutputStream? = null
try {
fos = FileOutputStream(dbFile, false)
val properties = SortedProperties()
loadedProps.forEach { (key, value) ->
when (key) {
"_salt" -> properties[key] = Sys.bytesToHex(value)
"_private" -> properties[key] = Sys.bytesToHex(value)
is InetAddress -> properties[IP.toString(key)] = Sys.bytesToHex(value)
else -> logger.error("Unable to parse property [$key] $value")
}
}
properties.store(fos, "Server salt, public/private keys, and remote computer public Keys")
fos.flush()
properties.clear()
lastModifiedTime = dbFile.lastModified()
} catch (e: IOException) {
logger.error("Properties cannot save to: $dbFile", e)
} finally {
if (fos != null) {
try {
fos.close()
} catch (ignored: IOException) {
}
}
}
}
override fun close() {
save()
}
}

5
src9/module-info.java
View File

@ -13,17 +13,18 @@ module dorkbox.network {
exports dorkbox.network.storage;
exports dorkbox.network.storage.types;
requires dorkbox.bytes;
requires dorkbox.updates;
requires dorkbox.utilities;
requires dorkbox.netutil;
requires dorkbox.minlog;
requires dorkbox.serializers;
// requires dorkbox.storage;
requires dorkbox.storage;
requires dorkbox.objectpool;
requires expiringmap;
requires net.jodah.typetools;
requires de.javakaffee.kryoserializers;
// requires de.javakaffee.kryoserializers;
requires com.esotericsoftware.kryo;
requires com.esotericsoftware.reflectasm;
requires org.objenesis;

9
test/dorkboxTest/network/AeronClient.kt
View File

@ -21,10 +21,12 @@ import ch.qos.logback.classic.encoder.PatternLayoutEncoder
import ch.qos.logback.classic.joran.JoranConfigurator
import ch.qos.logback.classic.spi.ILoggingEvent
import ch.qos.logback.core.ConsoleAppender
import dorkbox.netUtil.IPv4
import dorkbox.network.Client
import dorkbox.network.Configuration
import dorkbox.network.connection.Connection
import dorkbox.network.storage.types.MemoryStore
import dorkbox.network.ipFilter.IpSubnetFilterRule
import dorkbox.storage.Storage
import org.slf4j.LoggerFactory
import sun.misc.Unsafe
import java.lang.reflect.Field
@ -96,13 +98,12 @@ object AeronClient {
@JvmStatic
fun main(args: Array<String>) {
val configuration = Configuration()
configuration.settingsStore = MemoryStore.type() // don't want to persist anything on disk!
configuration.settingsStore = Storage.Memory() // don't want to persist anything on disk!
configuration.subscriptionPort = 2000
configuration.publicationPort = 2001
val client = Client<Connection>(configuration)
// client.filter(IpSubnetFilterRule(IPv4.LOCALHOST, 32, IpFilterRuleType.ACCEPT))
client.filter(IpSubnetFilterRule(IPv4.LOCALHOST, 32))
client.filter {
println("should this connection be allowed?")

4
test/dorkboxTest/network/AeronServer.kt
View File

@ -24,7 +24,7 @@ import ch.qos.logback.core.ConsoleAppender
import dorkbox.network.Server
import dorkbox.network.ServerConfiguration
import dorkbox.network.connection.Connection
import dorkbox.network.storage.types.MemoryStore
import dorkbox.storage.Storage
import org.slf4j.LoggerFactory
import sun.misc.Unsafe
import java.lang.reflect.Field
@ -72,7 +72,7 @@ object AeronServer {
@JvmStatic
fun main(args: Array<String>) {
val configuration = ServerConfiguration()
configuration.settingsStore = MemoryStore.type() // don't want to persist anything on disk!
configuration.settingsStore = Storage.Memory() // don't want to persist anything on disk!
configuration.listenIpAddress = "127.0.0.1"
configuration.subscriptionPort = 2000
configuration.publicationPort = 2001

6
test/dorkboxTest/network/BaseTest.kt
View File

@ -45,8 +45,8 @@ import dorkbox.network.Configuration
import dorkbox.network.Server
import dorkbox.network.ServerConfiguration
import dorkbox.network.connection.EndPoint
import dorkbox.network.storage.types.MemoryStore
import dorkbox.os.OS
import dorkbox.storage.Storage
import dorkbox.util.entropy.Entropy
import dorkbox.util.entropy.SimpleEntropy
import dorkbox.util.exceptions.InitializationException
@ -74,7 +74,7 @@ abstract class BaseTest {
fun clientConfig(block: Configuration.() -> Unit = {}): Configuration {
val configuration = Configuration()
configuration.settingsStore = MemoryStore.type() // don't want to persist anything on disk!
configuration.settingsStore = Storage.Memory() // don't want to persist anything on disk!
configuration.subscriptionPort = 2200
configuration.publicationPort = 2201
@ -86,7 +86,7 @@ abstract class BaseTest {
fun serverConfig(block: ServerConfiguration.() -> Unit = {}): ServerConfiguration {
val configuration = ServerConfiguration()
configuration.settingsStore = MemoryStore.type() // don't want to persist anything on disk!
configuration.settingsStore = Storage.Memory() // don't want to persist anything on disk!
configuration.subscriptionPort = 2200
configuration.publicationPort = 2201

29
test/dorkboxTest/network/StorageTest.kt
View File

@ -19,11 +19,7 @@ import dorkbox.network.Client
import dorkbox.network.Server
import dorkbox.network.connection.Connection
import dorkbox.network.storage.SettingsStore
import dorkbox.network.storage.StorageType
import dorkbox.network.storage.types.MemoryAccess
import dorkbox.network.storage.types.MemoryStore
import dorkbox.network.storage.types.PropertyStore
import mu.KLogger
import dorkbox.storage.Storage
import mu.KotlinLogging
import org.junit.Assert
import org.junit.Test
@ -33,14 +29,7 @@ class StorageTest : BaseTest() {
@Test
fun sharedStoreTest() {
// we want the server + client to have the SAME info
val store = MemoryAccess(KotlinLogging.logger("StorageType"))
val sharedStore = object : StorageType {
override fun create(logger: KLogger): SettingsStore {
return SettingsStore(logger, store)
}
}
val sharedStore = Storage.Memory().shared()
val serverConfig = serverConfig {
settingsStore = sharedStore
@ -63,10 +52,10 @@ class StorageTest : BaseTest() {
@Test
fun memoryTest() {
val salt1 = MemoryStore.type().create().use { it.getSalt() }
val salt1 = SettingsStore(Storage.Memory(), KotlinLogging.logger("test1")).use { it.getSalt() }
val salt2 = Server<Connection>(serverConfig().apply { settingsStore = MemoryStore.type() }).use { it.storage.getSalt() }
val salt3 = Server<Connection>(serverConfig().apply { settingsStore = MemoryStore.type() }).use { it.storage.getSalt() }
val salt2 = Server<Connection>(serverConfig().apply { settingsStore = Storage.Memory() }).use { it.storage.getSalt() }
val salt3 = Server<Connection>(serverConfig().apply { settingsStore = Storage.Memory() }).use { it.storage.getSalt() }
Assert.assertFalse(salt1.contentEquals(salt2))
Assert.assertFalse(salt1.contentEquals(salt3))
@ -98,14 +87,14 @@ class StorageTest : BaseTest() {
fun propFileTest() {
val file = File("test.db").absoluteFile
val salt1 = PropertyStore.type(file).create().use { it.getSalt() }
val salt2 = PropertyStore.type(file).create().use { it.getSalt() }
val salt1 = SettingsStore(Storage.Property(), KotlinLogging.logger("test1")).use { it.getSalt() }
val salt2 = SettingsStore(Storage.Property(), KotlinLogging.logger("test2")).use { it.getSalt() }
Assert.assertArrayEquals(salt1, salt2)
file.delete()
val salt3 = Server<Connection>(serverConfig().apply { settingsStore = PropertyStore.type(file) }).use { it.storage.getSalt() }
val salt4 = Server<Connection>(serverConfig().apply { settingsStore = PropertyStore.type(file) }).use { it.storage.getSalt() }
val salt3 = Server<Connection>(serverConfig().apply { settingsStore = Storage.Property().file(file) }).use { it.storage.getSalt() }
val salt4 = Server<Connection>(serverConfig().apply { settingsStore = Storage.Property().file(file) }).use { it.storage.getSalt() }
Assert.assertArrayEquals(salt3, salt4)
Assert.assertFalse(salt1.contentEquals(salt4))

4
test/dorkboxTest/network/rmi/multiJVM/TestClient.kt
View File

@ -24,7 +24,7 @@ import ch.qos.logback.classic.spi.ILoggingEvent
import ch.qos.logback.core.ConsoleAppender
import dorkbox.network.Client
import dorkbox.network.connection.Connection
import dorkbox.network.storage.types.MemoryStore
import dorkbox.storage.Storage
import dorkboxTest.network.BaseTest
import dorkboxTest.network.rmi.RmiCommonTest
import dorkboxTest.network.rmi.cows.TestCow
@ -67,7 +67,7 @@ object TestClient {
setup()
val config = BaseTest.clientConfig()
config.settingsStore = MemoryStore.type() // don't want to persist anything on disk!
config.settingsStore = Storage.Memory() // don't want to persist anything on disk!
config.enableRemoteSignatureValidation = false
config.enableIpc = false
config.aeronDirectoryForceUnique = true

4
test/dorkboxTest/network/rmi/multiJVM/TestServer.kt
View File

@ -17,7 +17,7 @@ package dorkboxTest.network.rmi.multiJVM
import dorkbox.network.Server
import dorkbox.network.connection.Connection
import dorkbox.network.storage.types.MemoryStore
import dorkbox.storage.Storage
import dorkboxTest.network.BaseTest
import dorkboxTest.network.rmi.cows.MessageWithTestCow
import dorkboxTest.network.rmi.cows.TestBabyCowImpl
@ -37,7 +37,7 @@ object TestServer {
val configuration = BaseTest.serverConfig()
configuration.enableRemoteSignatureValidation = false
configuration.settingsStore = MemoryStore.type() // don't want to persist anything on disk!
configuration.settingsStore = Storage.Memory() // don't want to persist anything on disk!
configuration.serialization.rmi.register(TestCow::class.java, TestCowImpl::class.java)
configuration.serialization.register(MessageWithTestCow::class.java)