diff --git a/src/dorkbox/network/Configuration.kt b/src/dorkbox/network/Configuration.kt index d6a7fb98..7d65879f 100644 --- a/src/dorkbox/network/Configuration.kt +++ b/src/dorkbox/network/Configuration.kt @@ -23,9 +23,8 @@ import dorkbox.network.aeron.CoroutineIdleStrategy import dorkbox.network.aeron.CoroutineSleepingMillisIdleStrategy import dorkbox.network.connection.Connection import dorkbox.network.serialization.Serialization -import dorkbox.network.storage.StorageType -import dorkbox.network.storage.types.PropertyStore import dorkbox.os.OS +import dorkbox.storage.Storage import io.aeron.driver.Configuration import io.aeron.driver.ThreadingMode import io.aeron.exceptions.DriverTimeoutException @@ -92,16 +91,8 @@ class ServerConfiguration : dorkbox.network.Configuration() { /** * Allows the user to change how endpoint settings and public key information are saved. - * - * For example, a custom database instead of the default, in-memory storage. - * - * Included types are: - * * ChronicleMapStore.type(file) -- high performance, but non-transactional and not recommended to be shared - * * LmdbStore.type(file) -- high performance, ACID, and can be shared - * * MemoryStore.type() -- v. high performance, but not persistent - * * PropertyStore.type(file) -- slow performance on write, but can easily be edited by user (similar to how openSSH server key info is) */ - override var settingsStore: StorageType = PropertyStore.type("settings-server.db") + override var settingsStore: Storage.Builder = Storage.Property().file("settings-server.db") set(value) { require(!contextDefined) { errorMessage } field = value @@ -265,17 +256,9 @@ open class Configuration { /** * Allows the user to change how endpoint settings and public key information are saved. * - * For example, a custom database instead of the default, in-memory storage. - * - * Included types are: - * * ChronicleMapStore.type(file) -- high performance, but non-transactional and not recommended to be shared - * * LmdbStore.type(file) -- high performance, ACID, and can be shared - * * MemoryStore.type() -- v. high performance, but not persistent - * * PropertyStore.type(file) -- slow performance on write, but can easily be edited by user (similar to how openSSH server key info is) - * * Note: This field is overridden for server configurations, so that the file used is different for client/server */ - open var settingsStore: StorageType = PropertyStore.type("settings-client.db") + open var settingsStore: Storage.Builder = Storage.Property().file("settings-client.db") set(value) { require(!contextDefined) { errorMessage } field = value diff --git a/src/dorkbox/network/storage/GenericStore.kt b/src/dorkbox/network/storage/GenericStore.kt deleted file mode 100644 index 328cfce3..00000000 --- a/src/dorkbox/network/storage/GenericStore.kt +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2020 dorkbox, llc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package dorkbox.network.storage - -interface GenericStore: AutoCloseable { - /** - * similar to map.get() - */ - operator fun get(key: Any): ByteArray? - - /** - * similar to map.set() - * - * Setting to NULL removes the value - */ - operator fun set(key: Any, bytes: ByteArray?) -} diff --git a/src/dorkbox/network/storage/SettingsStore.kt b/src/dorkbox/network/storage/SettingsStore.kt index e6bc2862..d487883f 100644 --- a/src/dorkbox/network/storage/SettingsStore.kt +++ b/src/dorkbox/network/storage/SettingsStore.kt @@ -15,10 +15,17 @@ */ package dorkbox.network.storage +import dorkbox.bytes.decodeBase58 +import dorkbox.bytes.encodeToBase58String +import dorkbox.netUtil.IP import dorkbox.netUtil.IPv4 import dorkbox.netUtil.IPv6 import dorkbox.network.connection.CryptoManagement +import dorkbox.serializers.SerializationDefaults +import dorkbox.storage.Storage import mu.KLogger +import java.net.Inet4Address +import java.net.Inet6Address import java.net.InetAddress import java.security.SecureRandom @@ -26,7 +33,7 @@ import java.security.SecureRandom * This class provides a way for the network stack to use a database of some sort. */ @Suppress("unused") -class SettingsStore(val logger: KLogger, val store: GenericStore) : AutoCloseable { +class SettingsStore(storageBuilder: Storage.Builder, val logger: KLogger) : AutoCloseable { companion object { /** * Address 0.0.0.0 or ::0 may be used as a source address for this host on this network. @@ -40,9 +47,83 @@ class SettingsStore(val logger: KLogger, val store: GenericStore) : AutoCloseabl internal const val privateKey = "_private" } + + + val store: Storage + init { + store = storageBuilder.logger(logger).apply { + if (!isStringBased) { + // have to load/save keys+values as strings + onLoad { key, value, load -> + // key/value will be strings for a string based storage system + key as String + value as String + + // we want the keys to be easy to read in case we are using string based storage + val xKey: Any? = when (key) { + saltKey, privateKey -> key + else -> { + IP.toAddress(key) + } + } + + if (xKey == null) { + logger.error("Unable to parse onLoad key property [$key] $value") + return@onLoad + } + + val xValue = value.decodeBase58() + load(xKey, xValue) + }.onSave { key, value, save -> + // we want the keys to be easy to read in case we are using string based storage + val xKey = when (key) { + saltKey, privateKey, Storage.versionTag -> key + is InetAddress -> IP.toString(key) + else -> null + } + + if (xKey == null) { + logger.error("Unable to parse onSave key property [$key] $value") + return@onSave + } + + val xValue = when(value) { + is Long -> value.toString() + is ByteArray -> value.encodeToBase58String() + else -> null + } + + if (xValue == null) { + logger.error("Unable to parse onSave value property [$key] $value") + return@onSave + } + + // all values are stored as bytes + save(xKey, xValue) + } + } else { + // everything is stored as bytes. We use a serializer instead to register types for easy serialization + onNewSerializer { + register(ByteArray::class.java) + register(Inet4Address::class.java, SerializationDefaults.inet4AddressSerializer) + register(Inet6Address::class.java, SerializationDefaults.inet6AddressSerializer) + } + } + }.build() + + + + + + + + + + // have to init salt - if (store[saltKey] == null) { + val currentValue: ByteArray? = store[saltKey] + if (currentValue == null) { val secureRandom = SecureRandom() // server salt is used to salt usernames and other various connection handshake parameters diff --git a/src/dorkbox/network/storage/StorageType.kt b/src/dorkbox/network/storage/StorageType.kt deleted file mode 100644 index 35addd9b..00000000 --- a/src/dorkbox/network/storage/StorageType.kt +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2020 dorkbox, llc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package dorkbox.network.storage - -import mu.KLogger -import mu.KotlinLogging - -interface StorageType { - /** - * Creates the custom storage type - */ - fun create(logger: KLogger = KotlinLogging.logger("StorageType")): SettingsStore -} diff --git a/src/dorkbox/network/storage/types/MemoryStore.kt b/src/dorkbox/network/storage/types/MemoryStore.kt deleted file mode 100644 index c3978ac4..00000000 --- a/src/dorkbox/network/storage/types/MemoryStore.kt +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2020 dorkbox, llc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package dorkbox.network.storage.types - -import dorkbox.network.storage.GenericStore -import dorkbox.network.storage.SettingsStore -import dorkbox.network.storage.StorageType -import mu.KLogger -import org.agrona.collections.Object2ObjectHashMap - -/** - * In-Memory store - */ -object MemoryStore { - fun type() = object : StorageType { - override fun create(logger: KLogger): SettingsStore { - return SettingsStore(logger, MemoryAccess(logger)) - } - } -} - - -class MemoryAccess(val logger: KLogger): GenericStore { - private val map = Object2ObjectHashMap() - - init { - logger.info("Memory storage initialized") - } - - override fun get(key: Any): ByteArray? { - return map[key] - } - - override fun set(key: Any, bytes: ByteArray?) { - map[key] = bytes - } - - override fun close() { - } -} - diff --git a/src/dorkbox/network/storage/types/PropertyStore.kt b/src/dorkbox/network/storage/types/PropertyStore.kt deleted file mode 100644 index f18f3a1c..00000000 --- a/src/dorkbox/network/storage/types/PropertyStore.kt +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2020 dorkbox, llc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package dorkbox.network.storage.types - -import dorkbox.netUtil.IP -import dorkbox.network.storage.GenericStore -import dorkbox.network.storage.SettingsStore -import dorkbox.network.storage.StorageType -import dorkbox.util.Sys -import dorkbox.util.properties.SortedProperties -import mu.KLogger -import org.agrona.collections.Object2ObjectHashMap -import java.io.File -import java.io.FileInputStream -import java.io.FileOutputStream -import java.io.IOException -import java.net.InetAddress -import java.util.* - -/** - * Java property files - */ -class PropertyStore(val dbFile: File, val logger: KLogger): GenericStore { - companion object { - fun type(dbFile: String) : StorageType { - return type(File(dbFile)) - } - - fun type(dbFile: File) = object : StorageType { - override fun create(logger: KLogger): SettingsStore { - return SettingsStore(logger, PropertyStore (dbFile, logger)) - } - } - } - - @Volatile - private var lastModifiedTime = 0L - private val loadedProps = Object2ObjectHashMap() - - init { - load() - logger.info("Property file storage initialized at: '$dbFile'") - } - - - private fun load() { - // if we cannot load, then we create a properties file. - if (!dbFile.canRead() && !dbFile.createNewFile()) { - throw IOException("Cannot create file") - } - - val input = FileInputStream(dbFile) - - try { - val properties = Properties() - properties.load(input) - lastModifiedTime = dbFile.lastModified() - - properties.entries.forEach { - val key = it.key as String - val value = it.value as String - - when (key) { - SettingsStore.saltKey -> loadedProps[SettingsStore.saltKey] = Sys.hexToBytes(value) - SettingsStore.privateKey -> loadedProps[SettingsStore.privateKey] = Sys.hexToBytes(value) - else -> { - val address: InetAddress? = IP.toAddress(key) - if (address != null) { - loadedProps[address] = Sys.hexToBytes(value) - } else { - logger.error("Unable to parse property file: $dbFile $key $value") - } - } - } - } - properties.clear() - } catch (e: IOException) { - logger.error("Cannot load properties!", e) - e.printStackTrace() - } finally { - input.close() - } - } - - override operator fun get(key: Any): ByteArray? { - // we want to check the last modified time when getting, because if we edit the on-disk file, we want to those changes - val lastModifiedTime = dbFile.lastModified() - if (this.lastModifiedTime != lastModifiedTime) { - // we want to reload the info - load() - } - - - val any = loadedProps[key] - if (any != null) { - return any - } - - return null - } - - /** - * Setting to NULL removes it - */ - override operator fun set(key: Any, bytes: ByteArray?) { - val hasChanged = if (bytes == null) { - loadedProps.remove(key) != null - } else { - val prev = loadedProps.put(key, bytes) - !prev.contentEquals(bytes) - } - - // every time we set info, we want to save it to disk (so the file on disk will ALWAYS be current, and so we can modify it as we choose) - if (hasChanged) { - save() - } - } - - fun save() { - var fos: FileOutputStream? = null - try { - fos = FileOutputStream(dbFile, false) - - val properties = SortedProperties() - - loadedProps.forEach { (key, value) -> - when (key) { - "_salt" -> properties[key] = Sys.bytesToHex(value) - "_private" -> properties[key] = Sys.bytesToHex(value) - is InetAddress -> properties[IP.toString(key)] = Sys.bytesToHex(value) - else -> logger.error("Unable to parse property [$key] $value") - } - } - - properties.store(fos, "Server salt, public/private keys, and remote computer public Keys") - fos.flush() - properties.clear() - lastModifiedTime = dbFile.lastModified() - } catch (e: IOException) { - logger.error("Properties cannot save to: $dbFile", e) - } finally { - if (fos != null) { - try { - fos.close() - } catch (ignored: IOException) { - } - } - } - } - - override fun close() { - save() - } -} - - diff --git a/src9/module-info.java b/src9/module-info.java index 7303b60f..712742b0 100644 --- a/src9/module-info.java +++ b/src9/module-info.java @@ -13,17 +13,18 @@ module dorkbox.network { exports dorkbox.network.storage; exports dorkbox.network.storage.types; + requires dorkbox.bytes; requires dorkbox.updates; requires dorkbox.utilities; requires dorkbox.netutil; requires dorkbox.minlog; requires dorkbox.serializers; -// requires dorkbox.storage; + requires dorkbox.storage; requires dorkbox.objectpool; requires expiringmap; requires net.jodah.typetools; - requires de.javakaffee.kryoserializers; +// requires de.javakaffee.kryoserializers; requires com.esotericsoftware.kryo; requires com.esotericsoftware.reflectasm; requires org.objenesis; diff --git a/test/dorkboxTest/network/AeronClient.kt b/test/dorkboxTest/network/AeronClient.kt index 79c762ec..23031636 100644 --- a/test/dorkboxTest/network/AeronClient.kt +++ b/test/dorkboxTest/network/AeronClient.kt @@ -21,10 +21,12 @@ import ch.qos.logback.classic.encoder.PatternLayoutEncoder import ch.qos.logback.classic.joran.JoranConfigurator import ch.qos.logback.classic.spi.ILoggingEvent import ch.qos.logback.core.ConsoleAppender +import dorkbox.netUtil.IPv4 import dorkbox.network.Client import dorkbox.network.Configuration import dorkbox.network.connection.Connection -import dorkbox.network.storage.types.MemoryStore +import dorkbox.network.ipFilter.IpSubnetFilterRule +import dorkbox.storage.Storage import org.slf4j.LoggerFactory import sun.misc.Unsafe import java.lang.reflect.Field @@ -96,13 +98,12 @@ object AeronClient { @JvmStatic fun main(args: Array) { val configuration = Configuration() - configuration.settingsStore = MemoryStore.type() // don't want to persist anything on disk! + configuration.settingsStore = Storage.Memory() // don't want to persist anything on disk! configuration.subscriptionPort = 2000 configuration.publicationPort = 2001 val client = Client(configuration) - -// client.filter(IpSubnetFilterRule(IPv4.LOCALHOST, 32, IpFilterRuleType.ACCEPT)) + client.filter(IpSubnetFilterRule(IPv4.LOCALHOST, 32)) client.filter { println("should this connection be allowed?") diff --git a/test/dorkboxTest/network/AeronServer.kt b/test/dorkboxTest/network/AeronServer.kt index 5d15aae1..b93ec3e3 100644 --- a/test/dorkboxTest/network/AeronServer.kt +++ b/test/dorkboxTest/network/AeronServer.kt @@ -24,7 +24,7 @@ import ch.qos.logback.core.ConsoleAppender import dorkbox.network.Server import dorkbox.network.ServerConfiguration import dorkbox.network.connection.Connection -import dorkbox.network.storage.types.MemoryStore +import dorkbox.storage.Storage import org.slf4j.LoggerFactory import sun.misc.Unsafe import java.lang.reflect.Field @@ -72,7 +72,7 @@ object AeronServer { @JvmStatic fun main(args: Array) { val configuration = ServerConfiguration() - configuration.settingsStore = MemoryStore.type() // don't want to persist anything on disk! + configuration.settingsStore = Storage.Memory() // don't want to persist anything on disk! configuration.listenIpAddress = "127.0.0.1" configuration.subscriptionPort = 2000 configuration.publicationPort = 2001 diff --git a/test/dorkboxTest/network/BaseTest.kt b/test/dorkboxTest/network/BaseTest.kt index 09ac89ce..d144a70c 100644 --- a/test/dorkboxTest/network/BaseTest.kt +++ b/test/dorkboxTest/network/BaseTest.kt @@ -45,8 +45,8 @@ import dorkbox.network.Configuration import dorkbox.network.Server import dorkbox.network.ServerConfiguration import dorkbox.network.connection.EndPoint -import dorkbox.network.storage.types.MemoryStore import dorkbox.os.OS +import dorkbox.storage.Storage import dorkbox.util.entropy.Entropy import dorkbox.util.entropy.SimpleEntropy import dorkbox.util.exceptions.InitializationException @@ -74,7 +74,7 @@ abstract class BaseTest { fun clientConfig(block: Configuration.() -> Unit = {}): Configuration { val configuration = Configuration() - configuration.settingsStore = MemoryStore.type() // don't want to persist anything on disk! + configuration.settingsStore = Storage.Memory() // don't want to persist anything on disk! configuration.subscriptionPort = 2200 configuration.publicationPort = 2201 @@ -86,7 +86,7 @@ abstract class BaseTest { fun serverConfig(block: ServerConfiguration.() -> Unit = {}): ServerConfiguration { val configuration = ServerConfiguration() - configuration.settingsStore = MemoryStore.type() // don't want to persist anything on disk! + configuration.settingsStore = Storage.Memory() // don't want to persist anything on disk! configuration.subscriptionPort = 2200 configuration.publicationPort = 2201 diff --git a/test/dorkboxTest/network/StorageTest.kt b/test/dorkboxTest/network/StorageTest.kt index e03ac63f..096e52b2 100644 --- a/test/dorkboxTest/network/StorageTest.kt +++ b/test/dorkboxTest/network/StorageTest.kt @@ -19,11 +19,7 @@ import dorkbox.network.Client import dorkbox.network.Server import dorkbox.network.connection.Connection import dorkbox.network.storage.SettingsStore -import dorkbox.network.storage.StorageType -import dorkbox.network.storage.types.MemoryAccess -import dorkbox.network.storage.types.MemoryStore -import dorkbox.network.storage.types.PropertyStore -import mu.KLogger +import dorkbox.storage.Storage import mu.KotlinLogging import org.junit.Assert import org.junit.Test @@ -33,14 +29,7 @@ class StorageTest : BaseTest() { @Test fun sharedStoreTest() { // we want the server + client to have the SAME info - val store = MemoryAccess(KotlinLogging.logger("StorageType")) - - val sharedStore = object : StorageType { - override fun create(logger: KLogger): SettingsStore { - return SettingsStore(logger, store) - } - } - + val sharedStore = Storage.Memory().shared() val serverConfig = serverConfig { settingsStore = sharedStore @@ -63,10 +52,10 @@ class StorageTest : BaseTest() { @Test fun memoryTest() { - val salt1 = MemoryStore.type().create().use { it.getSalt() } + val salt1 = SettingsStore(Storage.Memory(), KotlinLogging.logger("test1")).use { it.getSalt() } - val salt2 = Server(serverConfig().apply { settingsStore = MemoryStore.type() }).use { it.storage.getSalt() } - val salt3 = Server(serverConfig().apply { settingsStore = MemoryStore.type() }).use { it.storage.getSalt() } + val salt2 = Server(serverConfig().apply { settingsStore = Storage.Memory() }).use { it.storage.getSalt() } + val salt3 = Server(serverConfig().apply { settingsStore = Storage.Memory() }).use { it.storage.getSalt() } Assert.assertFalse(salt1.contentEquals(salt2)) Assert.assertFalse(salt1.contentEquals(salt3)) @@ -98,14 +87,14 @@ class StorageTest : BaseTest() { fun propFileTest() { val file = File("test.db").absoluteFile - val salt1 = PropertyStore.type(file).create().use { it.getSalt() } - val salt2 = PropertyStore.type(file).create().use { it.getSalt() } + val salt1 = SettingsStore(Storage.Property(), KotlinLogging.logger("test1")).use { it.getSalt() } + val salt2 = SettingsStore(Storage.Property(), KotlinLogging.logger("test2")).use { it.getSalt() } Assert.assertArrayEquals(salt1, salt2) file.delete() - val salt3 = Server(serverConfig().apply { settingsStore = PropertyStore.type(file) }).use { it.storage.getSalt() } - val salt4 = Server(serverConfig().apply { settingsStore = PropertyStore.type(file) }).use { it.storage.getSalt() } + val salt3 = Server(serverConfig().apply { settingsStore = Storage.Property().file(file) }).use { it.storage.getSalt() } + val salt4 = Server(serverConfig().apply { settingsStore = Storage.Property().file(file) }).use { it.storage.getSalt() } Assert.assertArrayEquals(salt3, salt4) Assert.assertFalse(salt1.contentEquals(salt4)) diff --git a/test/dorkboxTest/network/rmi/multiJVM/TestClient.kt b/test/dorkboxTest/network/rmi/multiJVM/TestClient.kt index fc541caf..539ba22e 100644 --- a/test/dorkboxTest/network/rmi/multiJVM/TestClient.kt +++ b/test/dorkboxTest/network/rmi/multiJVM/TestClient.kt @@ -24,7 +24,7 @@ import ch.qos.logback.classic.spi.ILoggingEvent import ch.qos.logback.core.ConsoleAppender import dorkbox.network.Client import dorkbox.network.connection.Connection -import dorkbox.network.storage.types.MemoryStore +import dorkbox.storage.Storage import dorkboxTest.network.BaseTest import dorkboxTest.network.rmi.RmiCommonTest import dorkboxTest.network.rmi.cows.TestCow @@ -67,7 +67,7 @@ object TestClient { setup() val config = BaseTest.clientConfig() - config.settingsStore = MemoryStore.type() // don't want to persist anything on disk! + config.settingsStore = Storage.Memory() // don't want to persist anything on disk! config.enableRemoteSignatureValidation = false config.enableIpc = false config.aeronDirectoryForceUnique = true diff --git a/test/dorkboxTest/network/rmi/multiJVM/TestServer.kt b/test/dorkboxTest/network/rmi/multiJVM/TestServer.kt index 21acb3d1..cdffa1c5 100644 --- a/test/dorkboxTest/network/rmi/multiJVM/TestServer.kt +++ b/test/dorkboxTest/network/rmi/multiJVM/TestServer.kt @@ -17,7 +17,7 @@ package dorkboxTest.network.rmi.multiJVM import dorkbox.network.Server import dorkbox.network.connection.Connection -import dorkbox.network.storage.types.MemoryStore +import dorkbox.storage.Storage import dorkboxTest.network.BaseTest import dorkboxTest.network.rmi.cows.MessageWithTestCow import dorkboxTest.network.rmi.cows.TestBabyCowImpl @@ -37,7 +37,7 @@ object TestServer { val configuration = BaseTest.serverConfig() configuration.enableRemoteSignatureValidation = false - configuration.settingsStore = MemoryStore.type() // don't want to persist anything on disk! + configuration.settingsStore = Storage.Memory() // don't want to persist anything on disk! configuration.serialization.rmi.register(TestCow::class.java, TestCowImpl::class.java) configuration.serialization.register(MessageWithTestCow::class.java)