From 400aa33a7a9376ae3f37989317aad634cb30fe52 Mon Sep 17 00:00:00 2001 From: Masashi Katsumata Date: Fri, 13 Nov 2026 00:01:40 -0800 Subject: [PATCH] Store large circle sets by place, so a zoomed-in tile reads only the blocks round it The circle file kept its records in list order, 512 to a block with 48 blocks cached, so a tile's circles lay in most of the file: on the national antenna set a z12 tile over New York needed 436 of 1,414 blocks. CircleTileSet now writes the file in Z-order of each circle's z12 cell (a stable radix sort of 24-bit keys, the same key as js-sdk-core). Indices stay the list's: the storage translates them, and the grid index, its box checks and forEachRecord walk the storage order. The GPU recording keeps the drawn-last circle per dot pixel and adds the circles in list order. --- .../core/shapetile/ShapeTileGpuDeviceTest.kt | 14 + .../core/circle/CircleTileFileStorage.kt | 84 ++-- .../core/circle/CircleTileGeometry.kt | 90 ++++- .../mapconductor/core/circle/CircleTileSet.kt | 249 ++++++------ .../core/shapetile/ShapeGridIndex.kt | 363 +++++++++-------- .../core/shapetile/ShapeTileRenderer.kt | 13 +- .../core/shapetile/ShapeTileStorage.kt | 215 ++++++---- .../core/circle/CircleTileFileStorageTest.kt | 368 +++++++++++------- 8 files changed, 849 insertions(+), 547 deletions(-) diff --git a/src/androidTest/java/com/mapconductor/core/shapetile/ShapeTileGpuDeviceTest.kt b/src/androidTest/java/com/mapconductor/core/shapetile/ShapeTileGpuDeviceTest.kt index 23ac6d7..98793fa 100644 --- a/src/androidTest/java/com/mapconductor/core/shapetile/ShapeTileGpuDeviceTest.kt +++ b/src/androidTest/java/com/mapconductor/core/shapetile/ShapeTileGpuDeviceTest.kt @@ -256,5 +256,19 @@ class ShapeTileGpuDeviceTest { val (gpu, cpu) = both(::renderer, x, y, z) assertTrue("z=$z", mismatch(gpu, cpu, 48) < 0.03) } + // Stored by place: the GPU reads them in storage order and must still draw + // in list order, the same picture as the canvas and as index order. + val order = StorageOrder.sortedBy(states.size) { StorageOrder.mortonCellKey(states[it]!!.latitude, states[it]!!.longitude) } + fun byPlace() = + ShapeTileRenderer(tileSize = 256, pixelScale = { 1f }, painter = CircleTilePainter).apply { + setStorage(CircleTileFileStorage.create(temporary.newFolder(), states.size, { states[it] }, order = order)) + } + for ((z, x, y) in listOf(Triple(1, 0, 0), Triple(4, 3, 6), Triple(6, 12, 25))) { + val (gpu, cpu) = both(::byPlace, x, y, z) + assertTrue("by place z=$z", mismatch(gpu, cpu, 48) < 0.03) + ShapeTileRenderer.useGpu = true + val plain = renderer().renderBitmap(x, y, z, 1)!! + assertEquals("by place z=$z: the same tile as index order", 0.0, mismatch(gpu, plain, 0), 0.0) + } } } diff --git a/src/main/java/com/mapconductor/core/circle/CircleTileFileStorage.kt b/src/main/java/com/mapconductor/core/circle/CircleTileFileStorage.kt index 9cca6b3..02acf9a 100644 --- a/src/main/java/com/mapconductor/core/circle/CircleTileFileStorage.kt +++ b/src/main/java/com/mapconductor/core/circle/CircleTileFileStorage.kt @@ -2,6 +2,7 @@ package com.mapconductor.core.circle import com.mapconductor.core.InternalMapConductorApi import com.mapconductor.core.shapetile.ShapeTileStorage +import com.mapconductor.core.shapetile.StorageOrder import java.io.EOFException import java.io.File import java.io.RandomAccessFile @@ -13,12 +14,17 @@ import kotlin.math.max * Packed immutable circle geometry, read through a bounded block cache. * No nationwide TileCircle or bounding-box object arrays are retained. * Small updates share the file; a bounded override map is compacted when full. + * + * With an [order], the records are stored by place ([storageOrder]): a tile's + * circles then lie in a few blocks instead of most of the file. Everything + * here still takes the list's indices. */ @InternalMapConductorApi internal class CircleTileFileStorage private constructor( private val file: SharedFile, override val size: Int, override val maxStroke: Float, + private val order: StorageOrder?, private val overrides: Map = emptyMap(), ) : ShapeTileStorage() { init { @@ -29,18 +35,25 @@ internal class CircleTileFileStorage private constructor( internal val cachedBytes: Int get() = file.cachedBytes internal val allocatedReadBuffers: Int get() = file.allocatedReadBuffers + override val storageOrder: StorageOrder? get() = order + + private fun positionOf(index: Int): Int = order?.positionOf?.get(index) ?: index + override fun get(index: Int): TileCircle? = when { index !in 0 until size -> null overrides.isNotEmpty() && overrides.containsKey(index) -> overrides[index] - else -> file.read(index) { bytes, offset -> decode(bytes, offset) } + else -> file.read(positionOf(index)) { bytes, offset -> decode(bytes, offset) } } /** - * Calls [action] for each of [indices] in order, with the record's bytes and - * offset in them, or with the overriding circle (bytes null). Records of one - * block are read under one lock, not one per circle: a nationwide tile reads + * Calls [action] for each of [indices], with the record's bytes and offset + * in them, or with the overriding circle (bytes null). Records of one block + * are read under one lock, not one per circle: a nationwide tile reads * hundreds of thousands, on several tile threads at once. + * + * The calls come in storage order, which is [indices]' order only without + * an [order]: `k` says which of [indices] each one is. */ internal fun forEachRecord( indices: IntArray, @@ -51,27 +64,29 @@ internal class CircleTileFileStorage private constructor( // tiles read the same file on other threads, and must not wait while // this one records its circles. val scratch = ByteBuffer.allocate(BLOCK_BYTES).order(ByteOrder.LITTLE_ENDIAN) - var k = 0 - while (k < indices.size) { - checkCancelled() + // Each one's stored position (high half) and k (low half), in storage order. + val sorted = LongArray(indices.size) + var count = 0 + for (k in indices.indices) { val index = indices[k] - if (index !in 0 until size) { - k++ - continue - } + if (index !in 0 until size) continue if (overrides.isNotEmpty() && overrides.containsKey(index)) { action.record(k, null, 0, overrides[index]) - k++ continue } - val block = index / RECORDS_PER_BLOCK + sorted[count++] = (positionOf(index).toLong() shl 32) or k.toLong() + } + if (order != null) sorted.sort(0, count) + var at = 0 + while (at < count) { + checkCancelled() + val block = (sorted[at] ushr 32).toInt() / RECORDS_PER_BLOCK file.copyBlock(block, scratch) - while (k < indices.size) { - val i = indices[k] - if (i / RECORDS_PER_BLOCK != block || i !in 0 until size) break - if (overrides.isNotEmpty() && overrides.containsKey(i)) break - action.record(k, scratch, i % RECORDS_PER_BLOCK * RECORD_BYTES, null) - k++ + while (at < count) { + val position = (sorted[at] ushr 32).toInt() + if (position / RECORDS_PER_BLOCK != block) break + action.record(sorted[at].toInt(), scratch, position % RECORDS_PER_BLOCK * RECORD_BYTES, null) + at++ } } } @@ -92,7 +107,7 @@ internal class CircleTileFileStorage private constructor( index !in 0 until size -> null overrides.isNotEmpty() && overrides.containsKey(index) -> overrides[index]?.bbox else -> - file.read(index) { bytes, p -> + file.read(positionOf(index)) { bytes, p -> if (bytes.getInt(p + 76) and 1 == 0) { null } else { @@ -119,12 +134,12 @@ internal class CircleTileFileStorage private constructor( } changed[index] = item } - if (!compact) return CircleTileFileStorage(file, size, widest, changed) + if (!compact) return CircleTileFileStorage(file, size, widest, order, changed) // Compact by streaming, never by materialising every decoded circle. - val next = create(file.path.parentFile!!, size, { get(it) }) + val next = create(file.path.parentFile!!, size, { get(it) }, order = order) try { - next.file.patch(updates, size) - val result = CircleTileFileStorage(next.file, size, widest) + next.file.patch(updates.map { (index, item) -> positionOf(index) to item }, size) + val result = CircleTileFileStorage(next.file, size, widest, order) next.release() return result } catch (error: Throwable) { @@ -223,6 +238,7 @@ internal class CircleTileFileStorage private constructor( fun write( size: Int, itemAt: (Int) -> TileCircle?, + order: StorageOrder?, checkCancelled: () -> Unit, ): Float { val header = @@ -237,9 +253,9 @@ internal class CircleTileFileStorage private constructor( while (header.hasRemaining()) channel.write(header) val bytes = ByteBuffer.allocate(BLOCK_BYTES).order(ByteOrder.LITTLE_ENDIAN) var widest = 0f - for (index in 0 until size) { - if (index % 4096 == 0) checkCancelled() - val item = itemAt(index) + for (position in 0 until size) { + if (position % 4096 == 0) checkCancelled() + val item = itemAt(order?.indexAt?.get(position) ?: position) encode(bytes, bytes.position(), item) bytes.position(bytes.position() + RECORD_BYTES) if (item != null) widest = max(widest, CircleTilePainter.strokeWidth(item)) @@ -254,17 +270,18 @@ internal class CircleTileFileStorage private constructor( return widest } + /** [updates] by stored position. */ @Synchronized fun patch( updates: List>, size: Int, ) { check(references == 1) // Only an unpublished freshly compacted file may be written. val bytes = ByteBuffer.allocate(RECORD_BYTES).order(ByteOrder.LITTLE_ENDIAN) - for ((index, item) in updates) { - if (index in 0 until size) { + for ((position, item) in updates) { + if (position in 0 until size) { bytes.clear() encode(bytes, 0, item) - var at = HEADER_BYTES + index.toLong() * RECORD_BYTES + var at = HEADER_BYTES + position.toLong() * RECORD_BYTES while (bytes.hasRemaining()) { val n = channel.write(bytes, at) check(n > 0) @@ -287,13 +304,16 @@ internal class CircleTileFileStorage private constructor( private const val MAX_OVERRIDES = 2048 private val liveFiles = HashSet() + /** [order]: the order to store them in (see the class); index order when null. */ fun create( directory: File, size: Int, itemAt: (Int) -> TileCircle?, checkCancelled: () -> Unit = { }, + order: StorageOrder? = null, ): CircleTileFileStorage { + require(order == null || order.indexAt.size == size) require(size >= 0) val path = synchronized(liveFiles) { @@ -312,8 +332,8 @@ internal class CircleTileFileStorage private constructor( var file: SharedFile? = null try { file = SharedFile(path) - val widest = file.write(size, itemAt, checkCancelled) - return CircleTileFileStorage(file, size, widest) + val widest = file.write(size, itemAt, order, checkCancelled) + return CircleTileFileStorage(file, size, widest, order) } catch (error: Throwable) { if (file != null diff --git a/src/main/java/com/mapconductor/core/circle/CircleTileGeometry.kt b/src/main/java/com/mapconductor/core/circle/CircleTileGeometry.kt index 055c4a7..73dca59 100644 --- a/src/main/java/com/mapconductor/core/circle/CircleTileGeometry.kt +++ b/src/main/java/com/mapconductor/core/circle/CircleTileGeometry.kt @@ -362,7 +362,8 @@ internal object CircleTilePainter : ShapeTileRenderer.Painter { val item = override ?: return@forEachRecord if (!isVisible(item)) return@forEachRecord val slot = subpixelSlot(item, scale, originX, originY, tileSize) - if (slot >= 0) lastDot[slot] = index + 1 + // The pixel keeps the circle drawn last; calls come in storage order. + if (slot >= 0 && index + 1 > lastDot[slot]) lastDot[slot] = index + 1 return@forEachRecord } if (bytes.getInt(p + 76) and 1 == 0) return@forEachRecord @@ -371,8 +372,10 @@ internal object CircleTilePainter : ShapeTileRenderer.Painter { val strokeWidth = bytes.getFloat(p + 72) if (!isVisibleColor(fill) && !(strokeWidth > 0f && isVisibleColor(stroke))) return@forEachRecord val slot = slotOf(bytes.getDouble(p + 24), bytes.getDouble(p + 32), bytes.getDouble(p + 40), bytes.getDouble(p + 48), bytes.getDouble(p + 56)) - if (slot >= 0) lastDot[slot] = index + 1 + if (slot >= 0 && index + 1 > lastDot[slot]) lastDot[slot] = index + 1 } + // Gathered, then added in draw order (k): the calls come in storage order. + val deferred = DeferredCircles() storage.forEachRecord(candidates, checkCancelled) { k, bytes, p, override -> val index = candidates[k] if (active != null && !active[index]) return@forEachRecord @@ -380,7 +383,7 @@ internal object CircleTilePainter : ShapeTileRenderer.Painter { val item = override ?: return@forEachRecord val slot = subpixelSlot(item, scale, originX, originY, tileSize) if (slot == -1 || (slot >= 0 && lastDot[slot] != index + 1)) return@forEachRecord - record(scene, item, simplifyCircle(item, scale), originX, originY, r) + deferred.addItem(k, item) return@forEachRecord } if (bytes.getInt(p + 76) and 1 == 0) return@forEachRecord @@ -404,8 +407,8 @@ internal object CircleTilePainter : ShapeTileRenderer.Painter { val ryS = (south - y) * scale val maxR = max(rx, max(ryN, ryS)) if (maxR < CIRCLE_DOT_RADIUS_PX) { - scene.addCircle( - ((floor(cx) - originX) * r).toFloat(), ((floor(cy) - originY) * r).toFloat(), r, r, + deferred.add( + k, ((floor(cx) - originX) * r).toFloat(), ((floor(cy) - originY) * r).toFloat(), r, r, if (fillVisible) fill else stroke, 0, kind = 1, ) return@forEachRecord @@ -414,12 +417,14 @@ internal object CircleTilePainter : ShapeTileRenderer.Painter { if (max(abs(ryN - rx), abs(ryS - rx)) > CIRCLE_ARC_TOLERANCE_PX) throw GlShapeScene.NeedsBatches() val outlined = strokeVisible && 2 * maxR >= MIN_OUTLINED_PX if (!fillVisible && !outlined) return@forEachRecord - scene.addCircle( - ((cx - originX) * r).toFloat(), ((cy - originY) * r).toFloat(), (rx * r).toFloat(), + deferred.add( + k, ((cx - originX) * r).toFloat(), ((cy - originY) * r).toFloat(), (rx * r).toFloat(), if (outlined) strokeWidth * r else 0f, if (fillVisible) fill else 0, if (outlined) stroke else 0, kind = 0, ) } + checkCancelled() + deferred.addTo(scene) { item -> record(scene, item, simplifyCircle(item, scale), originX, originY, r) } return true } @@ -470,3 +475,74 @@ internal object CircleTilePainter : ShapeTileRenderer.Painter { } } } + +/** + * Circles of one tile gathered in storage order, to be added to the scene in + * draw order (`k`, their place among the tile's candidates). Primitive arrays: + * a zoomed-out tile has tens of thousands. + */ +@InternalMapConductorApi +private class DeferredCircles { + private var count = 0 + private var ks = IntArray(1024) + private var floats = FloatArray(1024 * 4) + private var ints = IntArray(1024 * 3) + private val items = ArrayList>() + + fun add( + k: Int, + x: Float, + y: Float, + a: Float, + b: Float, + fill: Int, + stroke: Int, + kind: Int, + ) { + if (count == ks.size) { + ks = ks.copyOf(count * 2) + floats = floats.copyOf(count * 8) + ints = ints.copyOf(count * 6) + } + ks[count] = k + floats[count * 4] = x + floats[count * 4 + 1] = y + floats[count * 4 + 2] = a + floats[count * 4 + 3] = b + ints[count * 3] = fill + ints[count * 3 + 1] = stroke + ints[count * 3 + 2] = kind + count++ + } + + /** An overriding circle: recorded through the painter when its turn comes. */ + fun addItem( + k: Int, + item: TileCircle, + ) { + items.add(k to item) + } + + /** Adds them to [scene] by k; [recordItem] records the overriding circles in their turn. */ + fun addTo( + scene: GlShapeScene, + recordItem: (TileCircle) -> Unit, + ) { + // k (high half) and which entry (low half; the overriding circles after the others). + val order = LongArray(count + items.size) + for (i in 0 until count) order[i] = (ks[i].toLong() shl 32) or i.toLong() + for ((j, entry) in items.withIndex()) order[count + j] = (entry.first.toLong() shl 32) or (count + j).toLong() + order.sort() + for (key in order) { + val i = key.toInt() + if (i >= count) { + recordItem(items[i - count].second) + continue + } + scene.addCircle( + floats[i * 4], floats[i * 4 + 1], floats[i * 4 + 2], floats[i * 4 + 3], + ints[i * 3], ints[i * 3 + 1], kind = ints[i * 3 + 2], + ) + } + } +} diff --git a/src/main/java/com/mapconductor/core/circle/CircleTileSet.kt b/src/main/java/com/mapconductor/core/circle/CircleTileSet.kt index 13f3bcb..9675782 100644 --- a/src/main/java/com/mapconductor/core/circle/CircleTileSet.kt +++ b/src/main/java/com/mapconductor/core/circle/CircleTileSet.kt @@ -1,121 +1,128 @@ -package com.mapconductor.core.circle - -import com.mapconductor.core.ResourceProvider -import com.mapconductor.core.normalizeLng -import com.mapconductor.core.shapetile.ShapeTileRenderer -import com.mapconductor.core.shapetile.ShapeTileSet -import com.mapconductor.core.tileserver.LocalTileServer -import com.mapconductor.core.tileserver.TileServerRegistry -import java.io.File -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.SupervisorJob -import kotlinx.coroutines.currentCoroutineContext -import kotlinx.coroutines.ensureActive - -/** - * The circles of one map that are drawn as raster tiles, and their route on - * the local tile server. - * - * Provider-independent: a provider decides which circles to tile - * ([CircleTilingOptions.shouldUseTiles], [canTile]), hands them to - * [setCircles], and shows the returned URL template as a raster layer. - * In-place changes go through [update]; each one ends in a new template (the - * version in it is what makes the map ask again), reported to - * [onTemplateChanged]. Tiles the change did not touch are still cached by the - * renderer and come back without being drawn again. - * - * The same as [com.mapconductor.core.polyline.PolylineTileSet] for circles: - * both are [ShapeTileSet]s, with their own route, version and tile cache. - * Same shape as js-sdk-core's and ios-sdk-core's `CircleTileSet`. - */ -class CircleTileSet( - server: LocalTileServer = TileServerRegistry.get(), - scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default), -) { - internal val renderer: ShapeTileRenderer = - ShapeTileRenderer(tileSize = TILE_SIZE, painter = CircleTilePainter) - - internal val shapes: ShapeTileSet = - ShapeTileSet( - routePrefix = "circles", - server = server, - scope = scope, - renderer = renderer, - idOf = { it.id }, - // The native renderers stack an unset zIndex by the automatic one; so do the tiles. - zIndexOf = ::effectiveZIndex, - toTile = TileCircle::from, - canReuse = { state, tile -> tile.matches(state) }, - createStorage = { states -> - if (states.size <= 2000) { - null - } else { - val context = currentCoroutineContext() - val root = - runCatching { ResourceProvider.getAppContext().cacheDir } - .getOrElse { File(System.getProperty("java.io.tmpdir")) } - CircleTileFileStorage.create( - File(root, "mapconductor-circle-geometry"), states.size, - { TileCircle.from(states[it]) }, { context.ensureActive() }, - ) - } - }, - ) - - val routeId: String get() = shapes.routeId - - /** Called with the new template after an [update] or [remove]. */ - var onTemplateChanged: (suspend (String) -> Unit)? - get() = shapes.onTemplateChanged - set(value) { - shapes.onTemplateChanged = value - } - - val tileSize: Int get() = TILE_SIZE - - /** - * Registers [states] (replacing whatever was registered) and returns the - * URL template of their tiles. Draw order is the effective zIndex (an - * unset one is the automatic one from the centre), then the order given. - */ - suspend fun setCircles(states: List): String = shapes.setShapes(states) - - fun has(id: String): Boolean = shapes.has(id) - - /** - * One circle changed in place. A change of zIndex reorders the list and - * needs [setCircles] instead; everything else is applied here. - */ - fun update(state: CircleState) = shapes.update(state) - - /** Takes one circle out of the tiles. Only the tiles it was in are drawn again. */ - fun remove(id: String) = shapes.remove(id) - - fun template(): String = shapes.template() - - fun dispose() = shapes.dispose() - - companion object { - private const val TILE_SIZE = 256 - private const val MAX_LATITUDE = 85.05112878 - - /** - * Whether a circle can be drawn in a tile. One with no radius, or one - * that reaches a pole, the edge of the Mercator world or the - * antimeridian, is drawn by the map SDK. Geodesic or not, the rest can. - */ - fun canTile(state: CircleState): Boolean { - val radius = state.radiusMeters - if (!(radius > 0.0) || !radius.isFinite()) return false - val lat = state.center.latitude - val lngRaw = state.center.longitude - if (!lat.isFinite() || !lngRaw.isFinite()) return false - val extent = circleExtent(lat, radius, state.geodesic) - if (extent.overPole) return false - if (extent.north >= MAX_LATITUDE || extent.south <= -MAX_LATITUDE) return false - val lng = normalizeLng(lngRaw) - return lng - extent.dLng > -180.0 && lng + extent.dLng < 180.0 - } - } -} +package com.mapconductor.core.circle + +import com.mapconductor.core.ResourceProvider +import com.mapconductor.core.normalizeLng +import com.mapconductor.core.shapetile.ShapeTileRenderer +import com.mapconductor.core.shapetile.ShapeTileSet +import com.mapconductor.core.shapetile.StorageOrder +import com.mapconductor.core.tileserver.LocalTileServer +import com.mapconductor.core.tileserver.TileServerRegistry +import java.io.File +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.ensureActive + +/** + * The circles of one map that are drawn as raster tiles, and their route on + * the local tile server. + * + * Provider-independent: a provider decides which circles to tile + * ([CircleTilingOptions.shouldUseTiles], [canTile]), hands them to + * [setCircles], and shows the returned URL template as a raster layer. + * In-place changes go through [update]; each one ends in a new template (the + * version in it is what makes the map ask again), reported to + * [onTemplateChanged]. Tiles the change did not touch are still cached by the + * renderer and come back without being drawn again. + * + * The same as [com.mapconductor.core.polyline.PolylineTileSet] for circles: + * both are [ShapeTileSet]s, with their own route, version and tile cache. + * Same shape as js-sdk-core's and ios-sdk-core's `CircleTileSet`. + */ +class CircleTileSet( + server: LocalTileServer = TileServerRegistry.get(), + scope: CoroutineScope = CoroutineScope(SupervisorJob() + Dispatchers.Default), +) { + internal val renderer: ShapeTileRenderer = + ShapeTileRenderer(tileSize = TILE_SIZE, painter = CircleTilePainter) + + internal val shapes: ShapeTileSet = + ShapeTileSet( + routePrefix = "circles", + server = server, + scope = scope, + renderer = renderer, + idOf = { it.id }, + // The native renderers stack an unset zIndex by the automatic one; so do the tiles. + zIndexOf = ::effectiveZIndex, + toTile = TileCircle::from, + canReuse = { state, tile -> tile.matches(state) }, + createStorage = { states -> + if (states.size <= 2000) { + null + } else { + val context = currentCoroutineContext() + val root = + runCatching { ResourceProvider.getAppContext().cacheDir } + .getOrElse { File(System.getProperty("java.io.tmpdir")) } + // Stored by place, so that a tile reads the few blocks round it. + val order = + StorageOrder.sortedBy(states.size) { + val center = states[it].center + StorageOrder.mortonCellKey(center.latitude, normalizeLng(center.longitude)) + } + CircleTileFileStorage.create( + File(root, "mapconductor-circle-geometry"), states.size, + { TileCircle.from(states[it]) }, { context.ensureActive() }, order, + ) + } + }, + ) + + val routeId: String get() = shapes.routeId + + /** Called with the new template after an [update] or [remove]. */ + var onTemplateChanged: (suspend (String) -> Unit)? + get() = shapes.onTemplateChanged + set(value) { + shapes.onTemplateChanged = value + } + + val tileSize: Int get() = TILE_SIZE + + /** + * Registers [states] (replacing whatever was registered) and returns the + * URL template of their tiles. Draw order is the effective zIndex (an + * unset one is the automatic one from the centre), then the order given. + */ + suspend fun setCircles(states: List): String = shapes.setShapes(states) + + fun has(id: String): Boolean = shapes.has(id) + + /** + * One circle changed in place. A change of zIndex reorders the list and + * needs [setCircles] instead; everything else is applied here. + */ + fun update(state: CircleState) = shapes.update(state) + + /** Takes one circle out of the tiles. Only the tiles it was in are drawn again. */ + fun remove(id: String) = shapes.remove(id) + + fun template(): String = shapes.template() + + fun dispose() = shapes.dispose() + + companion object { + private const val TILE_SIZE = 256 + private const val MAX_LATITUDE = 85.05112878 + + /** + * Whether a circle can be drawn in a tile. One with no radius, or one + * that reaches a pole, the edge of the Mercator world or the + * antimeridian, is drawn by the map SDK. Geodesic or not, the rest can. + */ + fun canTile(state: CircleState): Boolean { + val radius = state.radiusMeters + if (!(radius > 0.0) || !radius.isFinite()) return false + val lat = state.center.latitude + val lngRaw = state.center.longitude + if (!lat.isFinite() || !lngRaw.isFinite()) return false + val extent = circleExtent(lat, radius, state.geodesic) + if (extent.overPole) return false + if (extent.north >= MAX_LATITUDE || extent.south <= -MAX_LATITUDE) return false + val lng = normalizeLng(lngRaw) + return lng - extent.dLng > -180.0 && lng + extent.dLng < 180.0 + } + } +} diff --git a/src/main/java/com/mapconductor/core/shapetile/ShapeGridIndex.kt b/src/main/java/com/mapconductor/core/shapetile/ShapeGridIndex.kt index 6e1f328..e776dd2 100644 --- a/src/main/java/com/mapconductor/core/shapetile/ShapeGridIndex.kt +++ b/src/main/java/com/mapconductor/core/shapetile/ShapeGridIndex.kt @@ -1,171 +1,192 @@ -package com.mapconductor.core.shapetile - -import com.mapconductor.core.InternalMapConductorApi -import com.mapconductor.core.polygon.IntList -import com.mapconductor.core.polygon.POLYGON_CELL -import com.mapconductor.core.polygon.POLYGON_MAX_CELLS -import java.util.BitSet -import kotlin.math.floor - -/** A multiresolution grid: wide shapes use coarser cells, not a global scan list. */ -@InternalMapConductorApi -internal class ShapeGridIndex private constructor( - val cells: HashMap, - /** Shapes that formerly needed a global scan; retained for diagnostics. */ - val large: IntArray, - private val levels: List, - private val count: Int, - private val legacyLarge: IntArray = IntArray(0), -) { - @InternalMapConductorApi - private class Level( - val size: Double, - val cells: HashMap, - val groups: HashMap>? = null, - ) - - @InternalMapConductorApi - constructor(cells: HashMap, large: IntArray) : this( - cells, large, listOf(Level(POLYGON_CELL, cells)), - maxOf(cells.values.maxOfOrNull { it.maxOrNull() ?: -1 } ?: -1, large.maxOrNull() ?: -1) + 1, - large, - ) - - fun query( - bboxOf: (Int) -> DoubleArray?, - minX: Double, - minY: Double, - maxX: Double, - maxY: Double, - ): IntArray { - val seen = BitSet(count) - val matches = BitSet(count) - - fun take(i: Int) { - seen.set(i) - } - for (i in legacyLarge) take(i) - for (level in levels) { - val cx0 = floor(minX / level.size).toLong() - val cx1 = floor(maxX / level.size).toLong() - val cy0 = floor(minY / level.size).toLong() - val cy1 = floor(maxY / level.size).toLong() - val area = (cx1 - cx0 + 1).toDouble() * (cy1 - cy0 + 1) - val groups = level.groups - if (area > 16_384 && groups != null) { - val gx0 = Math.floorDiv(cx0, 16L) - val gx1 = Math.floorDiv(cx1, 16L) - val gy0 = Math.floorDiv(cy0, 16L) - val gy1 = Math.floorDiv(cy1, 16L) - val groupArea = (gx1 - gx0 + 1).toDouble() * (gy1 - gy0 + 1) - if (groupArea > groups.size) { - for (bucket in level.cells.values) for (i in bucket) take(i) - } else { - for (gx in gx0..gx1) { - for (gy in gy0..gy1) { - val buckets = groups[cellKey(gx, gy)] ?: continue - for (bucket in buckets) for (i in bucket) take(i) - } - } - } - } else if (area > level.cells.size) { - // A zoomed-out tile must not enumerate millions of empty cells. - for (bucket in level.cells.values) for (i in bucket) take(i) - } else { - for (cx in cx0..cx1) { - for (cy in cy0..cy1) { - val bucket = level.cells[cellKey(cx, cy)] ?: continue - for (i in bucket) take(i) - } - } - } - } - // Read boxes in source order. File-backed geometry can then stream - // through its block cache instead of seeking in spatial bucket order. - var candidate = seen.nextSetBit(0) - while (candidate >= 0) { - val b = bboxOf(candidate) - if (b != null && b[0] <= maxX && b[2] >= minX && b[1] <= maxY && b[3] >= minY) matches.set(candidate) - candidate = seen.nextSetBit(candidate + 1) - } - // A bitset removes duplicates before bounding-box checks and produces - // draw order without sorting the same shape once for every cell it spans. - val out = IntArray(matches.cardinality()) - var next = matches.nextSetBit(0) - var at = 0 - while (next >= 0) { - out[at++] = next - next = matches.nextSetBit(next + 1) - } - return out - } - - @InternalMapConductorApi - companion object { - fun build( - count: Int, - bboxOf: (Int) -> DoubleArray?, - ): ShapeGridIndex { - val buckets = Array(7) { HashMap() } - val canGroup = BooleanArray(7) { true } - val large = IntList() - for (i in 0 until count) { - val b = bboxOf(i) ?: continue - var level = 0 - var size = POLYGON_CELL - - fun cellsWide() = - (floor(b[2] / size) - floor(b[0] / size) + 1) * - (floor(b[3] / size) - floor(b[1] / size) + 1) - if (cellsWide() > POLYGON_MAX_CELLS) large.add(i) - while (level < buckets.lastIndex && cellsWide() > 16) { - level++ - size *= 4 - } - val cx0 = floor(b[0] / size).toLong() - val cx1 = floor(b[2] / size).toLong() - val cy0 = floor(b[1] / size).toLong() - val cy1 = floor(b[3] / size).toLong() - // Group keys can be decoded only when Y is in the packed key's - // nonnegative range. Keep the original query for polar overhangs. - if (cy0 < 0 || cy1 >= 1_048_576L) canGroup[level] = false - for (cx in cx0..cx1) { - for (cy in cy0..cy1) { - buckets[level].getOrPut(cellKey(cx, cy)) { IntList() }.add(i) - } - } - } - val levels = - buckets.mapIndexed { level, map -> - val packed = HashMap(map.size * 2) - for ((key, list) in map) packed[key] = list.toArray() - val groups = - if (canGroup[level]) { - val grouped = HashMap>() - for ((key, values) in packed) { - val cx = Math.floorDiv(key, 1_048_576L) - val cy = Math.floorMod(key, 1_048_576L) - grouped - .getOrPut(cellKey(Math.floorDiv(cx, 16L), Math.floorDiv(cy, 16L))) { ArrayList() } - .add(values) - } - HashMap>().also { out -> - for ((key, values) in grouped) { - out[key] = - values.toTypedArray() - } - } - } else { - null - } - Level(POLYGON_CELL * (1 shl (level * 2)), packed, groups) - } - return ShapeGridIndex(levels[0].cells, large.toArray(), levels.filter { it.cells.isNotEmpty() }, count) - } - - fun cellKey( - cx: Long, - cy: Long, - ): Long = cx * 1_048_576L + cy - } -} +package com.mapconductor.core.shapetile + +import com.mapconductor.core.InternalMapConductorApi +import com.mapconductor.core.polygon.IntList +import com.mapconductor.core.polygon.POLYGON_CELL +import com.mapconductor.core.polygon.POLYGON_MAX_CELLS +import java.util.BitSet +import kotlin.math.floor + +/** A multiresolution grid: wide shapes use coarser cells, not a global scan list. */ +@InternalMapConductorApi +internal class ShapeGridIndex private constructor( + val cells: HashMap, + /** Shapes that formerly needed a global scan; retained for diagnostics. */ + val large: IntArray, + private val levels: List, + private val count: Int, + private val legacyLarge: IntArray = IntArray(0), +) { + @InternalMapConductorApi + private class Level( + val size: Double, + val cells: HashMap, + val groups: HashMap>? = null, + ) + + @InternalMapConductorApi + constructor(cells: HashMap, large: IntArray) : this( + cells, large, listOf(Level(POLYGON_CELL, cells)), + maxOf(cells.values.maxOfOrNull { it.maxOrNull() ?: -1 } ?: -1, large.maxOrNull() ?: -1) + 1, + large, + ) + + fun query( + bboxOf: (Int) -> DoubleArray?, + minX: Double, + minY: Double, + maxX: Double, + maxY: Double, + order: StorageOrder? = null, + ): IntArray { + val seen = BitSet(count) + val matches = BitSet(count) + + fun take(i: Int) { + seen.set(i) + } + for (i in legacyLarge) take(i) + for (level in levels) { + val cx0 = floor(minX / level.size).toLong() + val cx1 = floor(maxX / level.size).toLong() + val cy0 = floor(minY / level.size).toLong() + val cy1 = floor(maxY / level.size).toLong() + val area = (cx1 - cx0 + 1).toDouble() * (cy1 - cy0 + 1) + val groups = level.groups + if (area > 16_384 && groups != null) { + val gx0 = Math.floorDiv(cx0, 16L) + val gx1 = Math.floorDiv(cx1, 16L) + val gy0 = Math.floorDiv(cy0, 16L) + val gy1 = Math.floorDiv(cy1, 16L) + val groupArea = (gx1 - gx0 + 1).toDouble() * (gy1 - gy0 + 1) + if (groupArea > groups.size) { + for (bucket in level.cells.values) for (i in bucket) take(i) + } else { + for (gx in gx0..gx1) { + for (gy in gy0..gy1) { + val buckets = groups[cellKey(gx, gy)] ?: continue + for (bucket in buckets) for (i in bucket) take(i) + } + } + } + } else if (area > level.cells.size) { + // A zoomed-out tile must not enumerate millions of empty cells. + for (bucket in level.cells.values) for (i in bucket) take(i) + } else { + for (cx in cx0..cx1) { + for (cy in cy0..cy1) { + val bucket = level.cells[cellKey(cx, cy)] ?: continue + for (i in bucket) take(i) + } + } + } + } + // Read boxes in storage order. File-backed geometry can then stream + // through its block cache instead of seeking in spatial bucket order. + fun check(candidate: Int) { + val b = bboxOf(candidate) + if (b != null && b[0] <= maxX && b[2] >= minX && b[1] <= maxY && b[3] >= minY) matches.set(candidate) + } + if (order == null) { + var candidate = seen.nextSetBit(0) + while (candidate >= 0) { + check(candidate) + candidate = seen.nextSetBit(candidate + 1) + } + } else { + val positions = BitSet(count) + var candidate = seen.nextSetBit(0) + while (candidate >= 0) { + positions.set(order.positionOf[candidate]) + candidate = seen.nextSetBit(candidate + 1) + } + var position = positions.nextSetBit(0) + while (position >= 0) { + check(order.indexAt[position]) + position = positions.nextSetBit(position + 1) + } + } + // A bitset removes duplicates before bounding-box checks and produces + // draw order without sorting the same shape once for every cell it spans. + val out = IntArray(matches.cardinality()) + var next = matches.nextSetBit(0) + var at = 0 + while (next >= 0) { + out[at++] = next + next = matches.nextSetBit(next + 1) + } + return out + } + + @InternalMapConductorApi + companion object { + /** [bboxOf] is called in [order] (storage order) when given; buckets need no order. */ + fun build( + count: Int, + order: StorageOrder? = null, + bboxOf: (Int) -> DoubleArray?, + ): ShapeGridIndex { + val buckets = Array(7) { HashMap() } + val canGroup = BooleanArray(7) { true } + val large = IntList() + for (p in 0 until count) { + val i = order?.indexAt?.get(p) ?: p + val b = bboxOf(i) ?: continue + var level = 0 + var size = POLYGON_CELL + + fun cellsWide() = + (floor(b[2] / size) - floor(b[0] / size) + 1) * + (floor(b[3] / size) - floor(b[1] / size) + 1) + if (cellsWide() > POLYGON_MAX_CELLS) large.add(i) + while (level < buckets.lastIndex && cellsWide() > 16) { + level++ + size *= 4 + } + val cx0 = floor(b[0] / size).toLong() + val cx1 = floor(b[2] / size).toLong() + val cy0 = floor(b[1] / size).toLong() + val cy1 = floor(b[3] / size).toLong() + // Group keys can be decoded only when Y is in the packed key's + // nonnegative range. Keep the original query for polar overhangs. + if (cy0 < 0 || cy1 >= 1_048_576L) canGroup[level] = false + for (cx in cx0..cx1) { + for (cy in cy0..cy1) { + buckets[level].getOrPut(cellKey(cx, cy)) { IntList() }.add(i) + } + } + } + val levels = + buckets.mapIndexed { level, map -> + val packed = HashMap(map.size * 2) + for ((key, list) in map) packed[key] = list.toArray() + val groups = + if (canGroup[level]) { + val grouped = HashMap>() + for ((key, values) in packed) { + val cx = Math.floorDiv(key, 1_048_576L) + val cy = Math.floorMod(key, 1_048_576L) + grouped + .getOrPut(cellKey(Math.floorDiv(cx, 16L), Math.floorDiv(cy, 16L))) { ArrayList() } + .add(values) + } + HashMap>().also { out -> + for ((key, values) in grouped) { + out[key] = + values.toTypedArray() + } + } + } else { + null + } + Level(POLYGON_CELL * (1 shl (level * 2)), packed, groups) + } + return ShapeGridIndex(levels[0].cells, large.toArray(), levels.filter { it.cells.isNotEmpty() }, count) + } + + fun cellKey( + cx: Long, + cy: Long, + ): Long = cx * 1_048_576L + cy + } +} diff --git a/src/main/java/com/mapconductor/core/shapetile/ShapeTileRenderer.kt b/src/main/java/com/mapconductor/core/shapetile/ShapeTileRenderer.kt index c0daefc..83ceed4 100644 --- a/src/main/java/com/mapconductor/core/shapetile/ShapeTileRenderer.kt +++ b/src/main/java/com/mapconductor/core/shapetile/ShapeTileRenderer.kt @@ -212,7 +212,10 @@ internal class ShapeTileRenderer( ): IntArray { val m = acquireModel() ?: return IntArray(0) return try { - m.index.query({ index -> if (m.active?.get(index) == false) null else m.storage.bbox(index) }, minX, minY, maxX, maxY) + m.index.query( + { index -> if (m.active?.get(index) == false) null else m.storage.bbox(index) }, + minX, minY, maxX, maxY, m.storage.storageOrder, + ) } finally { m.storage.release() } @@ -263,9 +266,10 @@ internal class ShapeTileRenderer( ) { val index = try { + var built = 0 ShapeGridIndex - .build(storage.size) { - if (it % 4096 == 0) checkCancelled() + .build(storage.size, storage.storageOrder) { + if (built++ % 4096 == 0) checkCancelled() storage.bbox(it) }.also { checkCancelled() } } catch (error: Throwable) { @@ -302,7 +306,7 @@ internal class ShapeTileRenderer( } val replacement = current.storage.updated(updates) next = replacement - val index = if (reindex) ShapeGridIndex.build(replacement.size, replacement::bbox) else current.index + val index = if (reindex) ShapeGridIndex.build(replacement.size, replacement.storageOrder, replacement::bbox) else current.index replaceModel(Model(replacement, index, maxStroke, current.perZoom, current.active)) next = null synchronized(cacheLock) { @@ -500,6 +504,7 @@ internal class ShapeTileRenderer( (originY - pad) / scale, (originX + tileSize + pad) / scale, (originY + tileSize + pad) / scale, + m.storage.storageOrder, ) } diff --git a/src/main/java/com/mapconductor/core/shapetile/ShapeTileStorage.kt b/src/main/java/com/mapconductor/core/shapetile/ShapeTileStorage.kt index 9be58af..a15b579 100644 --- a/src/main/java/com/mapconductor/core/shapetile/ShapeTileStorage.kt +++ b/src/main/java/com/mapconductor/core/shapetile/ShapeTileStorage.kt @@ -1,72 +1,143 @@ -package com.mapconductor.core.shapetile - -import com.mapconductor.core.InternalMapConductorApi -import java.util.concurrent.atomic.AtomicInteger -import kotlin.math.max - -/** Immutable geometry snapshot. A renderer owns one reference; each reader leases another. */ -@InternalMapConductorApi -internal abstract class ShapeTileStorage { - private val references = AtomicInteger(1) - abstract val size: Int - abstract val maxStroke: Float - - abstract operator fun get(index: Int): T? - - abstract fun bbox(index: Int): DoubleArray? - - abstract fun updated(updates: List>): ShapeTileStorage - - fun retain() { - while (true) { - val count = references.get() - check(count > 0) { "Geometry storage already closed" } - if (references.compareAndSet(count, count + 1)) return - } - } - - fun release() { - val remaining = references.decrementAndGet() - check(remaining >= 0) - if (remaining == 0) close() - } - - protected open fun close() = Unit -} - -/** Existing polygon/polyline storage, and small circle collections. */ -@InternalMapConductorApi -internal class MemoryShapeTileStorage private constructor( - private val items: Array, - private val boxes: Array, - override val maxStroke: Float, - private val painter: ShapeTileRenderer.Painter, -) : ShapeTileStorage() { - constructor(items: List, painter: ShapeTileRenderer.Painter) : this( - items.toTypedArray(), - Array(items.size) { items[it]?.let(painter::bbox) }, - items.maxOfOrNull { it?.let(painter::strokeWidth) ?: 0f } ?: 0f, - painter, - ) - - override val size: Int get() = items.size - - @Suppress("UNCHECKED_CAST") - override fun get(index: Int): T? = items.getOrNull(index) as T? - - override fun bbox(index: Int): DoubleArray? = boxes.getOrNull(index) - - override fun updated(updates: List>): ShapeTileStorage { - val next = items.copyOf() - val nextBoxes = boxes.copyOf() - var widest = maxStroke - for ((index, item) in updates) { - if (index in next.indices) { - next[index] = item - nextBoxes[index] = item?.let(painter::bbox) - if (item != null) widest = max(widest, painter.strokeWidth(item)) - } - } - return MemoryShapeTileStorage(next, nextBoxes, widest, painter) - } -} +package com.mapconductor.core.shapetile + +import com.mapconductor.core.InternalMapConductorApi +import java.util.concurrent.atomic.AtomicInteger +import kotlin.math.max + +/** Immutable geometry snapshot. A renderer owns one reference; each reader leases another. */ +@InternalMapConductorApi +internal abstract class ShapeTileStorage { + private val references = AtomicInteger(1) + abstract val size: Int + abstract val maxStroke: Float + + abstract operator fun get(index: Int): T? + + abstract fun bbox(index: Int): DoubleArray? + + abstract fun updated(updates: List>): ShapeTileStorage + + /** + * How the shapes are laid out in storage, when not in index order: a large + * file-backed set is stored by place, so that a tile reads the few blocks + * round it. Indices stay the list's (draw order, updates, activation); + * whatever reads many shapes walks them in this order. + */ + open val storageOrder: StorageOrder? get() = null + + fun retain() { + while (true) { + val count = references.get() + check(count > 0) { "Geometry storage already closed" } + if (references.compareAndSet(count, count + 1)) return + } + } + + fun release() { + val remaining = references.decrementAndGet() + check(remaining >= 0) + if (remaining == 0) close() + } + + protected open fun close() = Unit +} + +/** Existing polygon/polyline storage, and small circle collections. */ +@InternalMapConductorApi +internal class MemoryShapeTileStorage private constructor( + private val items: Array, + private val boxes: Array, + override val maxStroke: Float, + private val painter: ShapeTileRenderer.Painter, +) : ShapeTileStorage() { + constructor(items: List, painter: ShapeTileRenderer.Painter) : this( + items.toTypedArray(), + Array(items.size) { items[it]?.let(painter::bbox) }, + items.maxOfOrNull { it?.let(painter::strokeWidth) ?: 0f } ?: 0f, + painter, + ) + + override val size: Int get() = items.size + + @Suppress("UNCHECKED_CAST") + override fun get(index: Int): T? = items.getOrNull(index) as T? + + override fun bbox(index: Int): DoubleArray? = boxes.getOrNull(index) + + override fun updated(updates: List>): ShapeTileStorage { + val next = items.copyOf() + val nextBoxes = boxes.copyOf() + var widest = maxStroke + for ((index, item) in updates) { + if (index in next.indices) { + next[index] = item + nextBoxes[index] = item?.let(painter::bbox) + if (item != null) widest = max(widest, painter.strokeWidth(item)) + } + } + return MemoryShapeTileStorage(next, nextBoxes, widest, painter) + } +} + +/** Stored position -> index ([indexAt]) and back ([positionOf]). */ +@InternalMapConductorApi +internal class StorageOrder( + val indexAt: IntArray, +) { + val positionOf: IntArray = + IntArray(indexAt.size).also { positions -> for (p in indexAt.indices) positions[indexAt[p]] = p } + + @InternalMapConductorApi + companion object { + /** + * The indices `0 until size` sorted by [keyOf] (24-bit keys), ties in + * index order: two stable 12-bit counting passes, since a comparison sort + * of a nationwide set takes most of a second on a phone. + */ + fun sortedBy( + size: Int, + keyOf: (Int) -> Int, + ): StorageOrder { + val keys = IntArray(size) { keyOf(it) and 0xffffff } + var from = IntArray(size) { it } + var to = IntArray(size) + val counts = IntArray(4096) + for (shift in intArrayOf(0, 12)) { + counts.fill(0) + for (i in 0 until size) counts[(keys[from[i]] ushr shift) and 4095]++ + var sum = 0 + for (b in 0 until 4096) { + val c = counts[b] + counts[b] = sum + sum += c + } + for (i in 0 until size) to[counts[(keys[from[i]] ushr shift) and 4095]++] = from[i] + val swap = from + from = to + to = swap + } + return StorageOrder(from) + } + + /** + * The z12 cell of a point in Z-order (Morton), 24 bits: cells next to + * each other mostly get keys next to each other. The same key as + * js-sdk-core's `mortonCellKey` and ios-sdk-core's. + */ + fun mortonCellKey( + latitude: Double, + longitude: Double, + ): Int { + val x = (longitude + 180.0) / 360.0 + val sin = kotlin.math.sin(latitude.coerceIn(-85.0, 85.0) * Math.PI / 180.0) + val y = 0.5 - kotlin.math.ln((1 + sin) / (1 - sin)) / (4 * Math.PI) + val cx = kotlin.math.floor(x * 4096).toInt().coerceIn(0, 4095) + val cy = kotlin.math.floor(y * 4096).toInt().coerceIn(0, 4095) + var key = 0 + for (bit in 0 until 12) { + key = key or (((cx ushr bit) and 1) shl (2 * bit)) or (((cy ushr bit) and 1) shl (2 * bit + 1)) + } + return key + } + } +} diff --git a/src/test/java/com/mapconductor/core/circle/CircleTileFileStorageTest.kt b/src/test/java/com/mapconductor/core/circle/CircleTileFileStorageTest.kt index 8c6e349..f63df05 100644 --- a/src/test/java/com/mapconductor/core/circle/CircleTileFileStorageTest.kt +++ b/src/test/java/com/mapconductor/core/circle/CircleTileFileStorageTest.kt @@ -1,140 +1,228 @@ -package com.mapconductor.core.circle - -import androidx.compose.ui.graphics.Color -import androidx.compose.ui.unit.dp -import com.mapconductor.core.features.GeoPoint -import org.junit.Assert.assertArrayEquals -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Assert.assertNotEquals -import org.junit.Assert.assertNotNull -import org.junit.Assert.assertNull -import org.junit.Assert.assertTrue -import org.junit.Assert.fail -import org.junit.Rule -import org.junit.Test -import org.junit.rules.TemporaryFolder -import java.io.IOException -import java.util.concurrent.CancellationException - -class CircleTileFileStorageTest { - @get:Rule val temporary = TemporaryFolder() - - private fun state( - index: Int = 0, - geodesic: Boolean = true, - ) = CircleState( - center = GeoPoint(42.0 + index * 0.00001, -71.0), radiusMeters = 500.0 + index, - geodesic = geodesic, fillColor = Color(0x802563EB), strokeColor = Color.Red, strokeWidth = 0.5.dp, - ) - - @Test fun exactGeometryRoundTripAcrossBlockBoundaries() { - val storage = - CircleTileFileStorage.create(temporary.root, 1025, { - if (it == 511) null else TileCircle.from(state(it, it % 2 == 0)) - }) - try { - assertEquals(16L + 80L * 1025, storage.cacheFile.length()) - for (index in listOf(0, 510, 512, 1023, 1024)) { - val source = state(index, index % 2 == 0) - val decoded = storage[index]!! - assertTrue(decoded.matches(source)) - assertArrayEquals(TileCircle.from(source)!!.bbox, storage.bbox(index), 0.0) - } - assertNull(storage[511]) - assertNull(storage.bbox(511)) - assertNull(storage[-1]) - assertNull(storage[1025]) - } finally { - storage.release() - } - assertTrue(temporary.root.listFiles()!!.isEmpty()) - } - - @Test fun blockCacheRemainsBoundedAndEvictedRecordsReload() { - val storage = CircleTileFileStorage.create(temporary.root, 512 * 60, { TileCircle.from(state(it / 512)) }) - try { - for (block in 0 until 60) assertTrue(storage[block * 512]!!.matches(state(block))) - assertTrue(storage.cachedBytes <= 48 * 512 * 80) - assertTrue(storage[0]!!.matches(state())) - assertEquals(48, storage.allocatedReadBuffers) - } finally { - storage.release() - } - } - - @Test fun snapshotsShareFilesWithoutSharingUpdatedValuesOrReaderLifetimes() { - val before = CircleTileFileStorage.create(temporary.root, 3, { TileCircle.from(state(it)) }) - val path = before.cacheFile - before.retain() // A tile request that outlives replacement of the model. - val changed = TileCircle.from(state(40, false))!! - val after = before.updated(listOf(0 to changed, 1 to null)) - before.release() // Retire the old renderer owner. - try { - assertTrue(before[0]!!.matches(state())) - assertTrue(after[0]!!.matches(state(40, false))) - assertNull(after[1]) - assertNotNull(before[1]) - after.release() - assertTrue(path.exists()) - } finally { - before.release() - } - assertFalse(path.exists()) - } - - @Test fun largeUpdateCompactsWithoutModifyingOldSnapshots() { - val before = CircleTileFileStorage.create(temporary.root, 2100, { TileCircle.from(state()) }) - val oldPath = before.cacheFile - val changed = TileCircle.from(state(20, false))!! - val after = before.updated(List(2050) { it to if (it == 10) null else changed }) as CircleTileFileStorage - try { - assertNotEquals(oldPath, after.cacheFile) - assertTrue(before[0]!!.matches(state())) - assertTrue(after[0]!!.matches(state(20, false))) - assertNull(after[10]) - assertTrue(after[2099]!!.matches(state())) - before.release() - assertFalse(oldPath.exists()) - assertNotNull(after[1]) - } finally { - after.release() - } - assertTrue(temporary.root.listFiles()!!.isEmpty()) - } - - @Test fun updatingExistingSlotsAtTheOverrideLimitStillSharesTheFile() { - val before = CircleTileFileStorage.create(temporary.root, 3000, { TileCircle.from(state()) }) - val changed = TileCircle.from(state(30, false))!! - val atLimit = before.updated(List(2048) { it to changed }) as CircleTileFileStorage - val after = atLimit.updated(listOf(0 to null, 1 to changed)) as CircleTileFileStorage - try { - assertEquals(before.cacheFile, after.cacheFile) - assertNull(after[0]) - assertNotNull(atLimit[0]) - assertTrue(after[1]!!.matches(state(30, false))) - } finally { before.release(); atLimit.release(); after.release() } - assertTrue(temporary.root.listFiles()!!.isEmpty()) - } - - @Test fun cancelledOrFailedWritesDeletePartialCacheFiles() { - val active = CircleTileFileStorage.create(temporary.root, 2, { TileCircle.from(state()) }) - try { - for (cancel in listOf(false, true)) { - try { - CircleTileFileStorage.create(temporary.root, 10000, { - if (!cancel && it == 600) throw IOException("simulated write source failure") - TileCircle.from(state()) - }, { if (cancel) throw CancellationException("cancelled") }) - fail("Expected write failure") - } catch (_: IOException) { - } catch (_: CancellationException) { - } - assertEquals(listOf(active.cacheFile), temporary.root.listFiles()!!.toList()) - assertNotNull(active[0]) - } - } finally { - active.release() - } - } -} +package com.mapconductor.core.circle + +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.unit.dp +import com.mapconductor.core.features.GeoPoint +import com.mapconductor.core.shapetile.ShapeGridIndex +import com.mapconductor.core.shapetile.StorageOrder +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Assert.fail +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.IOException +import java.util.concurrent.CancellationException + +class CircleTileFileStorageTest { + @get:Rule val temporary = TemporaryFolder() + + private fun state( + index: Int = 0, + geodesic: Boolean = true, + ) = CircleState( + center = GeoPoint(42.0 + index * 0.00001, -71.0), radiusMeters = 500.0 + index, + geodesic = geodesic, fillColor = Color(0x802563EB), strokeColor = Color.Red, strokeWidth = 0.5.dp, + ) + + @Test fun exactGeometryRoundTripAcrossBlockBoundaries() { + val storage = + CircleTileFileStorage.create(temporary.root, 1025, { + if (it == 511) null else TileCircle.from(state(it, it % 2 == 0)) + }) + try { + assertEquals(16L + 80L * 1025, storage.cacheFile.length()) + for (index in listOf(0, 510, 512, 1023, 1024)) { + val source = state(index, index % 2 == 0) + val decoded = storage[index]!! + assertTrue(decoded.matches(source)) + assertArrayEquals(TileCircle.from(source)!!.bbox, storage.bbox(index), 0.0) + } + assertNull(storage[511]) + assertNull(storage.bbox(511)) + assertNull(storage[-1]) + assertNull(storage[1025]) + } finally { + storage.release() + } + assertTrue(temporary.root.listFiles()!!.isEmpty()) + } + + @Test fun blockCacheRemainsBoundedAndEvictedRecordsReload() { + val storage = CircleTileFileStorage.create(temporary.root, 512 * 60, { TileCircle.from(state(it / 512)) }) + try { + for (block in 0 until 60) assertTrue(storage[block * 512]!!.matches(state(block))) + assertTrue(storage.cachedBytes <= 48 * 512 * 80) + assertTrue(storage[0]!!.matches(state())) + assertEquals(48, storage.allocatedReadBuffers) + } finally { + storage.release() + } + } + + @Test fun snapshotsShareFilesWithoutSharingUpdatedValuesOrReaderLifetimes() { + val before = CircleTileFileStorage.create(temporary.root, 3, { TileCircle.from(state(it)) }) + val path = before.cacheFile + before.retain() // A tile request that outlives replacement of the model. + val changed = TileCircle.from(state(40, false))!! + val after = before.updated(listOf(0 to changed, 1 to null)) + before.release() // Retire the old renderer owner. + try { + assertTrue(before[0]!!.matches(state())) + assertTrue(after[0]!!.matches(state(40, false))) + assertNull(after[1]) + assertNotNull(before[1]) + after.release() + assertTrue(path.exists()) + } finally { + before.release() + } + assertFalse(path.exists()) + } + + @Test fun largeUpdateCompactsWithoutModifyingOldSnapshots() { + val before = CircleTileFileStorage.create(temporary.root, 2100, { TileCircle.from(state()) }) + val oldPath = before.cacheFile + val changed = TileCircle.from(state(20, false))!! + val after = before.updated(List(2050) { it to if (it == 10) null else changed }) as CircleTileFileStorage + try { + assertNotEquals(oldPath, after.cacheFile) + assertTrue(before[0]!!.matches(state())) + assertTrue(after[0]!!.matches(state(20, false))) + assertNull(after[10]) + assertTrue(after[2099]!!.matches(state())) + before.release() + assertFalse(oldPath.exists()) + assertNotNull(after[1]) + } finally { + after.release() + } + assertTrue(temporary.root.listFiles()!!.isEmpty()) + } + + @Test fun updatingExistingSlotsAtTheOverrideLimitStillSharesTheFile() { + val before = CircleTileFileStorage.create(temporary.root, 3000, { TileCircle.from(state()) }) + val changed = TileCircle.from(state(30, false))!! + val atLimit = before.updated(List(2048) { it to changed }) as CircleTileFileStorage + val after = atLimit.updated(listOf(0 to null, 1 to changed)) as CircleTileFileStorage + try { + assertEquals(before.cacheFile, after.cacheFile) + assertNull(after[0]) + assertNotNull(atLimit[0]) + assertTrue(after[1]!!.matches(state(30, false))) + } finally { before.release(); atLimit.release(); after.release() } + assertTrue(temporary.root.listFiles()!!.isEmpty()) + } + + @Test fun cancelledOrFailedWritesDeletePartialCacheFiles() { + val active = CircleTileFileStorage.create(temporary.root, 2, { TileCircle.from(state()) }) + try { + for (cancel in listOf(false, true)) { + try { + CircleTileFileStorage.create(temporary.root, 10000, { + if (!cancel && it == 600) throw IOException("simulated write source failure") + TileCircle.from(state()) + }, { if (cancel) throw CancellationException("cancelled") }) + fail("Expected write failure") + } catch (_: IOException) { + } catch (_: CancellationException) { + } + assertEquals(listOf(active.cacheFile), temporary.root.listFiles()!!.toList()) + assertNotNull(active[0]) + } + } finally { + active.release() + } + } + + /** Every fourth one here, the rest far away and spread out: in index order the ones here span every block. */ + private fun scattered(count: Int) = + List(count) { i -> + if (i % 4 == 0) { + CircleState(center = GeoPoint(42.0 + i * 0.000001, -71.0), radiusMeters = 100.0 + i, fillColor = Color.Blue) + } else { + CircleState(center = GeoPoint(30.0 + (i % 50), -120.0 + (i % 97) * 0.5), radiusMeters = 100.0 + i, fillColor = Color.Red) + } + } + + private fun byPlace(states: List) = + StorageOrder.sortedBy(states.size) { StorageOrder.mortonCellKey(states[it].center.latitude, states[it].center.longitude) } + + @Test fun storedByPlaceReadsAndIndexesByTheListsIndices() { + val states = scattered(2048) + val order = byPlace(states) + val storage = CircleTileFileStorage.create(temporary.root, states.size, { TileCircle.from(states[it]) }, order = order) + try { + assertEquals(order, storage.storageOrder) + // The ones here, before anything else is read: contiguous in storage, + // so at most the two blocks they straddle (all four in index order), + // and each call says which it is. + val here = IntArray(512) { it * 4 } + val seen = BooleanArray(here.size) + storage.forEachRecord(here, {}) { k, bytes, offset, _ -> + seen[k] = true + assertEquals(TileCircle.from(states[here[k]])!!.bbox[0], bytes!!.getDouble(offset + 24) - bytes.getDouble(offset + 40), 1e-12) + } + assertTrue(seen.all { it }) + assertTrue("blocks read: ${storage.allocatedReadBuffers}", storage.allocatedReadBuffers <= 2) + val plain = CircleTileFileStorage.create(temporary.newFolder(), states.size, { TileCircle.from(states[it]) }) + try { + plain.forEachRecord(here, {}) { _, _, _, _ -> } + assertEquals("blocks read in index order", 4, plain.allocatedReadBuffers) + } finally { + plain.release() + } + for (i in states.indices) { + assertTrue("index $i", storage[i]!!.matches(states[i])) + assertArrayEquals(TileCircle.from(states[i])!!.bbox, storage.bbox(i), 0.0) + } + } finally { + storage.release() + } + } + + @Test fun storedByPlaceKeepsIndicesThroughUpdatesAndCompaction() { + val states = scattered(4096) + val storage = CircleTileFileStorage.create(temporary.root, states.size, { TileCircle.from(states[it]) }, order = byPlace(states)) + // More than the override limit: compacts into a new file in the same order. + val changed = (0 until 3000).map { it to TileCircle.from(CircleState(center = GeoPoint(10.0, 10.0 + it * 0.001), radiusMeters = 7.0)) } + val after = storage.updated(changed) + storage.release() + try { + assertNotEquals(null, after.storageOrder) + for (i in 0 until 3000) assertEquals("index $i", 7.0, after[i]!!.radiusMeters, 0.0) + for (i in 3000 until 4096) assertTrue("index $i", after[i]!!.matches(states[i])) + } finally { + after.release() + } + } + + @Test fun theGridAnswersTheSameInStorageOrder() { + val states = scattered(3000) + val boxes = states.map { TileCircle.from(it)!!.bbox } + val order = byPlace(states) + val plain = ShapeGridIndex.build(boxes.size) { boxes[it] } + val read = ArrayList() + val ordered = ShapeGridIndex.build(boxes.size, order) { read.add(it); boxes[it] } + assertEquals("built in storage order", order.indexAt.toList(), read) + for ((x, y, size) in listOf(Triple(0.3, 0.37, 0.001), Triple(0.0, 0.0, 1.0), Triple(0.16, 0.38, 0.05))) { + val inspected = ArrayList() + val answer = ordered.query({ inspected.add(it); boxes[it] }, x, y, x + size, y + size, order) + assertArrayEquals(plain.query({ boxes[it] }, x, y, x + size, y + size), answer) + assertEquals("boxes read in storage order", inspected.sortedBy { order.positionOf[it] }, inspected) + } + } + + @Test fun mortonKeysAndTheOrderAreStable() { + // Cell (2048, 2048): bit 11 of x and of y. + assertEquals((1 shl 22) or (1 shl 23), StorageOrder.mortonCellKey(0.0, 0.0)) + val order = StorageOrder.sortedBy(6) { intArrayOf(5, 1, 5, 0, 1, 5)[it] } + assertArrayEquals(intArrayOf(3, 1, 4, 0, 2, 5), order.indexAt) + assertArrayEquals(intArrayOf(3, 1, 4, 0, 2, 5).let { a -> IntArray(6).also { p -> a.forEachIndexed { pos, i -> p[i] = pos } } }, order.positionOf) + } +}