487 lines
21 KiB
Kotlin
487 lines
21 KiB
Kotlin
///*-
|
|
// * #%L
|
|
// * LmdbJava
|
|
// * %%
|
|
// * Copyright (C) 2016 - 2020 The LmdbJava Open Source Project
|
|
// * %%
|
|
// * Licensed under the Apache License, Version 2.0 (the "License");
|
|
// * you may not use this file except in compliance with the License.
|
|
// * You may obtain a copy of the License at
|
|
// *
|
|
// * http://www.apache.org/licenses/LICENSE-2.0
|
|
// *
|
|
// * Unless required by applicable law or agreed to in writing, software
|
|
// * distributed under the License is distributed on an "AS IS" BASIS,
|
|
// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// * See the License for the specific language governing permissions and
|
|
// * limitations under the License.
|
|
// * #L%
|
|
// */
|
|
package dorkboxTest.network.lmdb
|
|
//
|
|
//import org.agrona.MutableDirectBuffer
|
|
//import org.agrona.concurrent.UnsafeBuffer
|
|
//import org.hamcrest.CoreMatchers
|
|
//import org.hamcrest.MatcherAssert
|
|
//import org.junit.Assert
|
|
//import org.junit.Ignore
|
|
//import org.junit.Rule
|
|
//import org.junit.Test
|
|
//import org.junit.rules.TemporaryFolder
|
|
//import org.lmdbjava.ByteBufferProxy
|
|
//import org.lmdbjava.DbiFlags
|
|
//import org.lmdbjava.DirectBufferProxy
|
|
//import org.lmdbjava.Env
|
|
//import org.lmdbjava.GetOp
|
|
//import org.lmdbjava.KeyRange
|
|
//import org.lmdbjava.SeekOp
|
|
//import org.lmdbjava.Verifier
|
|
//import java.io.File
|
|
//import java.io.IOException
|
|
//import java.nio.ByteBuffer
|
|
//import java.nio.charset.StandardCharsets
|
|
//import java.util.concurrent.Executors
|
|
//import java.util.concurrent.TimeUnit
|
|
//
|
|
///**
|
|
// * Welcome to LmdbJava!
|
|
// *
|
|
// *
|
|
// *
|
|
// * This short tutorial will walk you through using LmdbJava step-by-step.
|
|
// *
|
|
// *
|
|
// *
|
|
// * If you are using a 64-bit Windows, Linux or OS X machine, you can simply run
|
|
// * this tutorial by adding the LmdbJava JAR to your classpath. It includes the
|
|
// * required system libraries. If you are using another 64-bit platform, you'll
|
|
// * need to install the LMDB system library yourself. 32-bit platforms are not
|
|
// * supported.
|
|
// */
|
|
//@Ignore
|
|
//class TutorialTest {
|
|
//
|
|
// private val folder = TemporaryFolder()
|
|
//
|
|
// @Rule
|
|
// fun tmp(): TemporaryFolder {
|
|
// return folder
|
|
// }
|
|
//
|
|
// /**
|
|
// * In this first tutorial we will use LmdbJava with some basic defaults.
|
|
// *
|
|
// * @throws IOException if a path was unavailable for memory mapping
|
|
// */
|
|
// @Test
|
|
// @Throws(IOException::class)
|
|
// fun tutorial1() {
|
|
// // We need a storage directory first.
|
|
// // The path cannot be on a remote file system.
|
|
// val path = tmp().newFolder()
|
|
//
|
|
// // We always need an Env. An Env owns a physical on-disk storage file. One
|
|
// // Env can store many different databases (ie sorted maps).
|
|
// val env = Env.create() // LMDB also needs to know how large our DB might be. Over-estimating is OK.
|
|
// .setMapSize(10485760) // LMDB also needs to know how many DBs (Dbi) we want to store in this Env.
|
|
// .setMaxDbs(1) // Now let's open the Env. The same path can be concurrently opened and
|
|
// // used in different processes, but do not open the same path twice in
|
|
// // the same process at the same time.
|
|
// .open(path)
|
|
//
|
|
// // We need a Dbi for each DB. A Dbi roughly equates to a sorted map. The
|
|
// // MDB_CREATE flag causes the DB to be created if it doesn't already exist.
|
|
// val db = env.openDbi(DB_NAME, DbiFlags.MDB_CREATE)
|
|
//
|
|
// // We want to store some data, so we will need a direct ByteBuffer.
|
|
// // Note that LMDB keys cannot exceed maxKeySize bytes (511 bytes by default).
|
|
// // Values can be larger.
|
|
// val key = ByteBuffer.allocateDirect(env.maxKeySize)
|
|
// val `val` = ByteBuffer.allocateDirect(700)
|
|
// key.put("greeting".toByteArray(StandardCharsets.UTF_8)).flip()
|
|
// `val`.put("Hello world".toByteArray(StandardCharsets.UTF_8)).flip()
|
|
// val valSize = `val`.remaining()
|
|
//
|
|
// // Now store it. Dbi.put() internally begins and commits a transaction (Txn).
|
|
// db.put(key, `val`)
|
|
//
|
|
// // To fetch any data from LMDB we need a Txn. A Txn is very important in
|
|
// // LmdbJava because it offers ACID characteristics and internally holds a
|
|
// // read-only key buffer and read-only value buffer. These read-only buffers
|
|
// // are always the same two Java objects, but point to different LMDB-managed
|
|
// // memory as we use Dbi (and Cursor) methods. These read-only buffers remain
|
|
// // valid only until the Txn is released or the next Dbi or Cursor call. If
|
|
// // you need data afterwards, you should copy the bytes to your own buffer.
|
|
// env.txnRead().use { txn ->
|
|
// val found = db[txn, key]
|
|
// Assert.assertNotNull(found)
|
|
//
|
|
// // The fetchedVal is read-only and points to LMDB memory
|
|
// val fetchedVal = txn.`val`()
|
|
// MatcherAssert.assertThat(fetchedVal.remaining(), CoreMatchers.`is`(valSize))
|
|
//
|
|
// // Let's double-check the fetched value is correct
|
|
// MatcherAssert.assertThat(StandardCharsets.UTF_8.decode(fetchedVal).toString(), CoreMatchers.`is`("Hello world"))
|
|
// }
|
|
//
|
|
// // We can also delete. The simplest way is to let Dbi allocate a new Txn...
|
|
// db.delete(key)
|
|
// env.txnRead().use { txn -> Assert.assertNull(db[txn, key]) }
|
|
// env.close()
|
|
// }
|
|
//
|
|
// /**
|
|
// * In this second tutorial we'll learn more about LMDB's ACID Txns.
|
|
// *
|
|
// * @throws IOException if a path was unavailable for memory mapping
|
|
// * @throws InterruptedException if executor shutdown interrupted
|
|
// */
|
|
// @Test
|
|
// @Throws(IOException::class, InterruptedException::class)
|
|
// fun tutorial2() {
|
|
// val env = createSimpleEnv(tmp().newFolder())
|
|
// val db = env.openDbi(DB_NAME, DbiFlags.MDB_CREATE)
|
|
// val key = ByteBuffer.allocateDirect(env.maxKeySize)
|
|
// val `val` = ByteBuffer.allocateDirect(700)
|
|
//
|
|
// // Let's write and commit "key1" via a Txn. A Txn can include multiple Dbis.
|
|
// // Note write Txns block other write Txns, due to writes being serialized.
|
|
// // It's therefore important to avoid unnecessarily long-lived write Txns.
|
|
// env.txnWrite().use { txn ->
|
|
// key.put("key1".toByteArray(StandardCharsets.UTF_8)).flip()
|
|
// `val`.put("lmdb".toByteArray(StandardCharsets.UTF_8)).flip()
|
|
// db.put(txn, key, `val`)
|
|
//
|
|
// // We can read data too, even though this is a write Txn.
|
|
// val found = db[txn, key]
|
|
// Assert.assertNotNull(found)
|
|
//
|
|
// // An explicit commit is required, otherwise Txn.close() rolls it back.
|
|
// txn.commit()
|
|
// }
|
|
//
|
|
// // Open a read-only Txn. It only sees data that existed at Txn creation time.
|
|
// val rtx = env.txnRead()
|
|
//
|
|
// // Our read Txn can fetch key1 without problem, as it existed at Txn creation.
|
|
// var found = db[rtx, key]
|
|
// Assert.assertNotNull(found)
|
|
//
|
|
// // Note that our main test thread holds the Txn. Only one Txn per thread is
|
|
// // typically permitted (the exception is a read-only Env with MDB_NOTLS).
|
|
// //
|
|
// // Let's write out a "key2" via a new write Txn in a different thread.
|
|
// val es = Executors.newCachedThreadPool()
|
|
// es.execute {
|
|
// env.txnWrite().use { txn ->
|
|
// key.clear()
|
|
// key.put("key2".toByteArray(StandardCharsets.UTF_8)).flip()
|
|
// db.put(txn, key, `val`)
|
|
// txn.commit()
|
|
// }
|
|
// }
|
|
// es.shutdown()
|
|
// es.awaitTermination(10, TimeUnit.SECONDS)
|
|
//
|
|
// // Even though key2 has been committed, our read Txn still can't see it.
|
|
// found = db[rtx, key]
|
|
// Assert.assertNull(found)
|
|
//
|
|
// // To see key2, we could create a new Txn. But a reset/renew is much faster.
|
|
// // Reset/renew is also important to avoid long-lived read Txns, as these
|
|
// // prevent the re-use of free pages by write Txns (ie the DB will grow).
|
|
// rtx.reset()
|
|
// // ... potentially long operation here ...
|
|
// rtx.renew()
|
|
// found = db[rtx, key]
|
|
// Assert.assertNotNull(found)
|
|
//
|
|
// // Don't forget to close the read Txn now we're completely finished. We could
|
|
// // have avoided this if we used a try-with-resources block, but we wanted to
|
|
// // play around with multiple concurrent Txns to demonstrate the "I" in ACID.
|
|
// rtx.close()
|
|
// env.close()
|
|
// }
|
|
//
|
|
// /**
|
|
// * In this third tutorial we'll have a look at the Cursor. Up until now we've
|
|
// * just used Dbi, which is good enough for simple cases but unsuitable if you
|
|
// * don't know the key to fetch, or want to iterate over all the data etc.
|
|
// *
|
|
// * @throws IOException if a path was unavailable for memory mapping
|
|
// */
|
|
// @Test
|
|
// @Throws(IOException::class)
|
|
// fun tutorial3() {
|
|
// val env = createSimpleEnv(tmp().newFolder())
|
|
// val db = env.openDbi(DB_NAME, DbiFlags.MDB_CREATE)
|
|
// val key = ByteBuffer.allocateDirect(env.maxKeySize)
|
|
// val `val` = ByteBuffer.allocateDirect(700)
|
|
// env.txnWrite().use { txn ->
|
|
// // A cursor always belongs to a particular Dbi.
|
|
// val c = db.openCursor(txn)
|
|
//
|
|
// // We can put via a Cursor. Note we're adding keys in a strange order,
|
|
// // as we want to show you that LMDB returns them in sorted order.
|
|
// key.put("zzz".toByteArray(StandardCharsets.UTF_8)).flip()
|
|
// `val`.put("lmdb".toByteArray(StandardCharsets.UTF_8)).flip()
|
|
// c.put(key, `val`)
|
|
// key.clear()
|
|
// key.put("aaa".toByteArray(StandardCharsets.UTF_8)).flip()
|
|
// c.put(key, `val`)
|
|
// key.clear()
|
|
// key.put("ccc".toByteArray(StandardCharsets.UTF_8)).flip()
|
|
// c.put(key, `val`)
|
|
//
|
|
// // We can read from the Cursor by key.
|
|
// c[key, GetOp.MDB_SET]
|
|
// MatcherAssert.assertThat(StandardCharsets.UTF_8.decode(c.key()).toString(), CoreMatchers.`is`("ccc"))
|
|
//
|
|
// // Let's see that LMDB provides the keys in appropriate order....
|
|
// c.seek(SeekOp.MDB_FIRST)
|
|
// MatcherAssert.assertThat(StandardCharsets.UTF_8.decode(c.key()).toString(), CoreMatchers.`is`("aaa"))
|
|
//
|
|
// c.seek(SeekOp.MDB_LAST)
|
|
// MatcherAssert.assertThat(StandardCharsets.UTF_8.decode(c.key()).toString(), CoreMatchers.`is`("zzz"))
|
|
//
|
|
// c.seek(SeekOp.MDB_PREV)
|
|
// MatcherAssert.assertThat(StandardCharsets.UTF_8.decode(c.key()).toString(), CoreMatchers.`is`("ccc"))
|
|
//
|
|
// // Cursors can also delete the current key.
|
|
// c.delete()
|
|
//
|
|
// c.close()
|
|
// txn.commit()
|
|
// }
|
|
//
|
|
// // A read-only Cursor can survive its original Txn being closed. This is
|
|
// // useful if you want to close the original Txn (eg maybe you created the
|
|
// // Cursor during the constructor of a singleton with a throw-away Txn). Of
|
|
// // course, you cannot use the Cursor if its Txn is closed or currently reset.
|
|
// val tx1 = env.txnRead()
|
|
// val c = db.openCursor(tx1)
|
|
// tx1.close()
|
|
//
|
|
// // The Cursor becomes usable again by "renewing" it with an active read Txn.
|
|
// val tx2 = env.txnRead()
|
|
// c.renew(tx2)
|
|
// c.seek(SeekOp.MDB_FIRST)
|
|
//
|
|
// // As usual with read Txns, we can reset and renew them. The Cursor does
|
|
// // not need any special handling if we do this.
|
|
// tx2.reset()
|
|
//
|
|
// // ... potentially long operation here ...
|
|
// tx2.renew()
|
|
// c.seek(SeekOp.MDB_LAST)
|
|
// tx2.close()
|
|
// env.close()
|
|
// }
|
|
//
|
|
// /**
|
|
// * In this fourth tutorial we'll take a quick look at the iterators. These are
|
|
// * a more Java idiomatic form of using the Cursors we looked at in tutorial 3.
|
|
// *
|
|
// * @throws IOException if a path was unavailable for memory mapping
|
|
// */
|
|
// @Test
|
|
// @Throws(IOException::class)
|
|
// fun tutorial4() {
|
|
// val env = createSimpleEnv(tmp().newFolder())
|
|
// val db = env.openDbi(DB_NAME, DbiFlags.MDB_CREATE)
|
|
// env.txnWrite().use { txn ->
|
|
// val key = ByteBuffer.allocateDirect(env.maxKeySize)
|
|
// val `val` = ByteBuffer.allocateDirect(700)
|
|
//
|
|
// // Insert some data. Note that ByteBuffer order defaults to Big Endian.
|
|
// // LMDB does not persist the byte order, but it's critical to sort keys.
|
|
// // If your numeric keys don't sort as expected, review buffer byte order.
|
|
// `val`.putInt(100)
|
|
// key.putInt(1)
|
|
// db.put(txn, key, `val`)
|
|
// key.clear()
|
|
// key.putInt(2)
|
|
// db.put(txn, key, `val`)
|
|
// key.clear()
|
|
//
|
|
// // Each iterable uses a cursor and must be closed when finished. Iterate
|
|
// // forward in terms of key ordering starting with the first key.
|
|
// db.iterate(txn, KeyRange.all()).use { ci ->
|
|
// for (kv in ci) {
|
|
// MatcherAssert.assertThat(kv.key(), CoreMatchers.notNullValue())
|
|
// MatcherAssert.assertThat(kv.`val`(), CoreMatchers.notNullValue())
|
|
// }
|
|
// }
|
|
//
|
|
// // Iterate backward in terms of key ordering starting with the last key.
|
|
// db.iterate(txn, KeyRange.allBackward()).use { ci ->
|
|
// for (kv in ci) {
|
|
// MatcherAssert.assertThat(kv.key(), CoreMatchers.notNullValue())
|
|
// MatcherAssert.assertThat(kv.`val`(), CoreMatchers.notNullValue())
|
|
// }
|
|
// }
|
|
//
|
|
// // There are many ways to control the desired key range via KeyRange, such
|
|
// // as arbitrary start and stop values, direction etc. We've adopted Guava's
|
|
// // terminology for our range classes (see KeyRangeType for further details).
|
|
// key.putInt(1)
|
|
// val range = KeyRange.atLeastBackward(key)
|
|
// db.iterate(txn, range).use { ci ->
|
|
// for (kv in ci) {
|
|
// MatcherAssert.assertThat(kv.key(), CoreMatchers.notNullValue())
|
|
// MatcherAssert.assertThat(kv.`val`(), CoreMatchers.notNullValue())
|
|
// }
|
|
// }
|
|
// }
|
|
//
|
|
//
|
|
// env.close()
|
|
// }
|
|
//
|
|
// /**
|
|
// * In this fifth tutorial we'll explore multiple values sharing a single key.
|
|
// *
|
|
// * @throws IOException if a path was unavailable for memory mapping
|
|
// */
|
|
// @Test
|
|
// @Throws(IOException::class)
|
|
// fun tutorial5() {
|
|
// val env = createSimpleEnv(tmp().newFolder())
|
|
//
|
|
// // This time we're going to tell the Dbi it can store > 1 value per key.
|
|
// // There are other flags available if we're storing integers etc.
|
|
// val db = env.openDbi(DB_NAME, DbiFlags.MDB_CREATE, DbiFlags.MDB_DUPSORT)
|
|
//
|
|
// // Duplicate support requires both keys and values to be <= max key size.
|
|
// val key = ByteBuffer.allocateDirect(env.maxKeySize)
|
|
// val `val` = ByteBuffer.allocateDirect(env.maxKeySize)
|
|
//
|
|
// env.txnWrite().use { txn ->
|
|
// val c = db.openCursor(txn)
|
|
//
|
|
// // Store one key, but many values, and in non-natural order.
|
|
// key.put("key".toByteArray(StandardCharsets.UTF_8)).flip()
|
|
// `val`.put("xxx".toByteArray(StandardCharsets.UTF_8)).flip()
|
|
// c.put(key, `val`)
|
|
// `val`.clear()
|
|
// `val`.put("kkk".toByteArray(StandardCharsets.UTF_8)).flip()
|
|
// c.put(key, `val`)
|
|
// `val`.clear()
|
|
// `val`.put("lll".toByteArray(StandardCharsets.UTF_8)).flip()
|
|
// c.put(key, `val`)
|
|
//
|
|
// // Cursor can tell us how many values the current key has.
|
|
// val count = c.count()
|
|
// MatcherAssert.assertThat(count, CoreMatchers.`is`(3L))
|
|
//
|
|
// // Let's position the Cursor. Note sorting still works.
|
|
// c.seek(SeekOp.MDB_FIRST)
|
|
// MatcherAssert.assertThat(StandardCharsets.UTF_8.decode(c.`val`()).toString(), CoreMatchers.`is`("kkk"))
|
|
//
|
|
// c.seek(SeekOp.MDB_LAST)
|
|
// MatcherAssert.assertThat(StandardCharsets.UTF_8.decode(c.`val`()).toString(), CoreMatchers.`is`("xxx"))
|
|
//
|
|
// c.seek(SeekOp.MDB_PREV)
|
|
// MatcherAssert.assertThat(StandardCharsets.UTF_8.decode(c.`val`()).toString(), CoreMatchers.`is`("lll"))
|
|
//
|
|
// c.close()
|
|
// txn.commit()
|
|
// }
|
|
//
|
|
// env.close()
|
|
// }
|
|
//
|
|
// /**
|
|
// * Next up we'll show you how to easily check your platform (operating system
|
|
// * and Java version) is working properly with LmdbJava and the embedded LMDB
|
|
// * native library.
|
|
// *
|
|
// * @throws IOException if a path was unavailable for memory mapping
|
|
// */
|
|
// @Test
|
|
// @Throws(IOException::class)
|
|
// fun tutorial6() {
|
|
// // Note we need to specify the Verifier's DBI_COUNT for the Env.
|
|
// val env = Env.create(ByteBufferProxy.PROXY_OPTIMAL).setMapSize(10485760).setMaxDbs(Verifier.DBI_COUNT).open(tmp().newFolder())
|
|
//
|
|
// // Create a Verifier (it's a Callable<Long> for those needing full control).
|
|
// val v = Verifier(env)
|
|
//
|
|
// // We now run the verifier for 3 seconds; it raises an exception on failure.
|
|
// // The method returns the number of entries it successfully verified.
|
|
// v.runFor(3, TimeUnit.SECONDS)
|
|
// env.close()
|
|
// }
|
|
//
|
|
// /**
|
|
// * In this final tutorial we'll look at using Agrona's DirectBuffer.
|
|
// *
|
|
// * @throws IOException if a path was unavailable for memory mapping
|
|
// */
|
|
// @Test
|
|
// @Throws(IOException::class)
|
|
// fun tutorial7() {
|
|
// // The critical difference is we pass the PROXY_DB field to Env.create().
|
|
// // There's also a PROXY_SAFE if you want to stop ByteBuffer's Unsafe use.
|
|
// // Aside from that and a different type argument, it's the same as usual...
|
|
// val env = Env.create(DirectBufferProxy.PROXY_DB).setMapSize(10485760).setMaxDbs(1).open(tmp().newFolder())
|
|
//
|
|
// val db = env.openDbi(DB_NAME, DbiFlags.MDB_CREATE)
|
|
//
|
|
// val keyBb = ByteBuffer.allocateDirect(env.maxKeySize)
|
|
// val key: MutableDirectBuffer = UnsafeBuffer(keyBb)
|
|
// val `val`: MutableDirectBuffer = UnsafeBuffer(ByteBuffer.allocateDirect(700))
|
|
//
|
|
// env.txnWrite().use { txn ->
|
|
// db.openCursor(txn).use { c ->
|
|
// // Agrona is faster than ByteBuffer and its methods are nicer...
|
|
// `val`.putStringWithoutLengthUtf8(0, "The Value")
|
|
// key.putStringWithoutLengthUtf8(0, "yyy")
|
|
// c.put(key, `val`)
|
|
//
|
|
// key.putStringWithoutLengthUtf8(0, "ggg")
|
|
// c.put(key, `val`)
|
|
//
|
|
// c.seek(SeekOp.MDB_FIRST)
|
|
// MatcherAssert.assertThat(c.key().getStringWithoutLengthUtf8(0, env.maxKeySize), CoreMatchers.startsWith("ggg"))
|
|
//
|
|
// c.seek(SeekOp.MDB_LAST)
|
|
// MatcherAssert.assertThat(c.key().getStringWithoutLengthUtf8(0, env.maxKeySize), CoreMatchers.startsWith("yyy"))
|
|
//
|
|
// // DirectBuffer has no position concept. Often you don't want to store
|
|
// // the unnecessary bytes of a varying-size buffer. Let's have a look...
|
|
// val keyLen = key.putStringWithoutLengthUtf8(0, "12characters")
|
|
// MatcherAssert.assertThat(keyLen, CoreMatchers.`is`(12))
|
|
// MatcherAssert.assertThat(key.capacity(), CoreMatchers.`is`(env.maxKeySize))
|
|
//
|
|
// // To only store the 12 characters, we simply call wrap:
|
|
// key.wrap(key, 0, keyLen)
|
|
// MatcherAssert.assertThat(key.capacity(), CoreMatchers.`is`(keyLen))
|
|
// c.put(key, `val`)
|
|
// c.seek(SeekOp.MDB_FIRST)
|
|
// MatcherAssert.assertThat(c.key().capacity(), CoreMatchers.`is`(keyLen))
|
|
// MatcherAssert.assertThat(c.key().getStringWithoutLengthUtf8(0, c.key().capacity()), CoreMatchers.`is`("12characters"))
|
|
//
|
|
// // To store bigger values again, just wrap the original buffer.
|
|
// key.wrap(keyBb)
|
|
// MatcherAssert.assertThat(key.capacity(), CoreMatchers.`is`(env.maxKeySize))
|
|
// }
|
|
// txn.commit()
|
|
// }
|
|
//
|
|
// env.close()
|
|
// }
|
|
//
|
|
// // You've finished! There are lots of other neat things we could show you (eg
|
|
// // how to speed up inserts by appending them in key order, using integer
|
|
// // or reverse ordered keys, using Env.DISABLE_CHECKS_PROP etc), but you now
|
|
// // know enough to tackle the JavaDocs with confidence. Have fun!
|
|
// private fun createSimpleEnv(path: File): Env<ByteBuffer> {
|
|
// return Env.create().setMapSize(10485760).setMaxDbs(1).setMaxReaders(1).open(path)
|
|
// }
|
|
//
|
|
// companion object {
|
|
// private const val DB_NAME = "my DB"
|
|
// }
|
|
//}
|