Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -256,5 +256,19 @@ class ShapeTileGpuDeviceTest {
val (gpu, cpu) = both(::renderer, x, y, z)
assertTrue("z=$z", mismatch(gpu, cpu, 48) < 0.03)
}
// Stored by place: the GPU reads them in storage order and must still draw
// in list order, the same picture as the canvas and as index order.
val order = StorageOrder.sortedBy(states.size) { StorageOrder.mortonCellKey(states[it]!!.latitude, states[it]!!.longitude) }
fun byPlace() =
ShapeTileRenderer(tileSize = 256, pixelScale = { 1f }, painter = CircleTilePainter).apply {
setStorage(CircleTileFileStorage.create(temporary.newFolder(), states.size, { states[it] }, order = order))
}
for ((z, x, y) in listOf(Triple(1, 0, 0), Triple(4, 3, 6), Triple(6, 12, 25))) {
val (gpu, cpu) = both(::byPlace, x, y, z)
assertTrue("by place z=$z", mismatch(gpu, cpu, 48) < 0.03)
ShapeTileRenderer.useGpu = true
val plain = renderer().renderBitmap(x, y, z, 1)!!
assertEquals("by place z=$z: the same tile as index order", 0.0, mismatch(gpu, plain, 0), 0.0)
}
}
}
84 changes: 52 additions & 32 deletions src/main/java/com/mapconductor/core/circle/CircleTileFileStorage.kt
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package com.mapconductor.core.circle

import com.mapconductor.core.InternalMapConductorApi
import com.mapconductor.core.shapetile.ShapeTileStorage
import com.mapconductor.core.shapetile.StorageOrder
import java.io.EOFException
import java.io.File
import java.io.RandomAccessFile
Expand All @@ -13,12 +14,17 @@ import kotlin.math.max
* Packed immutable circle geometry, read through a bounded block cache.
* No nationwide TileCircle or bounding-box object arrays are retained.
* Small updates share the file; a bounded override map is compacted when full.
*
* With an [order], the records are stored by place ([storageOrder]): a tile's
* circles then lie in a few blocks instead of most of the file. Everything
* here still takes the list's indices.
*/
@InternalMapConductorApi
internal class CircleTileFileStorage private constructor(
private val file: SharedFile,
override val size: Int,
override val maxStroke: Float,
private val order: StorageOrder?,
private val overrides: Map<Int, TileCircle?> = emptyMap(),
) : ShapeTileStorage<TileCircle>() {
init {
Expand All @@ -29,18 +35,25 @@ internal class CircleTileFileStorage private constructor(
internal val cachedBytes: Int get() = file.cachedBytes
internal val allocatedReadBuffers: Int get() = file.allocatedReadBuffers

override val storageOrder: StorageOrder? get() = order

private fun positionOf(index: Int): Int = order?.positionOf?.get(index) ?: index

override fun get(index: Int): TileCircle? =
when {
index !in 0 until size -> null
overrides.isNotEmpty() && overrides.containsKey(index) -> overrides[index]
else -> file.read(index) { bytes, offset -> decode(bytes, offset) }
else -> file.read(positionOf(index)) { bytes, offset -> decode(bytes, offset) }
}

/**
* Calls [action] for each of [indices] in order, with the record's bytes and
* offset in them, or with the overriding circle (bytes null). Records of one
* block are read under one lock, not one per circle: a nationwide tile reads
* Calls [action] for each of [indices], with the record's bytes and offset
* in them, or with the overriding circle (bytes null). Records of one block
* are read under one lock, not one per circle: a nationwide tile reads
* hundreds of thousands, on several tile threads at once.
*
* The calls come in storage order, which is [indices]' order only without
* an [order]: `k` says which of [indices] each one is.
*/
internal fun forEachRecord(
indices: IntArray,
Expand All @@ -51,27 +64,29 @@ internal class CircleTileFileStorage private constructor(
// tiles read the same file on other threads, and must not wait while
// this one records its circles.
val scratch = ByteBuffer.allocate(BLOCK_BYTES).order(ByteOrder.LITTLE_ENDIAN)
var k = 0
while (k < indices.size) {
checkCancelled()
// Each one's stored position (high half) and k (low half), in storage order.
val sorted = LongArray(indices.size)
var count = 0
for (k in indices.indices) {
val index = indices[k]
if (index !in 0 until size) {
k++
continue
}
if (index !in 0 until size) continue
if (overrides.isNotEmpty() && overrides.containsKey(index)) {
action.record(k, null, 0, overrides[index])
k++
continue
}
val block = index / RECORDS_PER_BLOCK
sorted[count++] = (positionOf(index).toLong() shl 32) or k.toLong()
}
if (order != null) sorted.sort(0, count)
var at = 0
while (at < count) {
checkCancelled()
val block = (sorted[at] ushr 32).toInt() / RECORDS_PER_BLOCK
file.copyBlock(block, scratch)
while (k < indices.size) {
val i = indices[k]
if (i / RECORDS_PER_BLOCK != block || i !in 0 until size) break
if (overrides.isNotEmpty() && overrides.containsKey(i)) break
action.record(k, scratch, i % RECORDS_PER_BLOCK * RECORD_BYTES, null)
k++
while (at < count) {
val position = (sorted[at] ushr 32).toInt()
if (position / RECORDS_PER_BLOCK != block) break
action.record(sorted[at].toInt(), scratch, position % RECORDS_PER_BLOCK * RECORD_BYTES, null)
at++
}
}
}
Expand All @@ -92,7 +107,7 @@ internal class CircleTileFileStorage private constructor(
index !in 0 until size -> null
overrides.isNotEmpty() && overrides.containsKey(index) -> overrides[index]?.bbox
else ->
file.read(index) { bytes, p ->
file.read(positionOf(index)) { bytes, p ->
if (bytes.getInt(p + 76) and 1 == 0) {
null
} else {
Expand All @@ -119,12 +134,12 @@ internal class CircleTileFileStorage private constructor(
}
changed[index] = item
}
if (!compact) return CircleTileFileStorage(file, size, widest, changed)
if (!compact) return CircleTileFileStorage(file, size, widest, order, changed)
// Compact by streaming, never by materialising every decoded circle.
val next = create(file.path.parentFile!!, size, { get(it) })
val next = create(file.path.parentFile!!, size, { get(it) }, order = order)
try {
next.file.patch(updates, size)
val result = CircleTileFileStorage(next.file, size, widest)
next.file.patch(updates.map { (index, item) -> positionOf(index) to item }, size)
val result = CircleTileFileStorage(next.file, size, widest, order)
next.release()
return result
} catch (error: Throwable) {
Expand Down Expand Up @@ -223,6 +238,7 @@ internal class CircleTileFileStorage private constructor(
fun write(
size: Int,
itemAt: (Int) -> TileCircle?,
order: StorageOrder?,
checkCancelled: () -> Unit,
): Float {
val header =
Expand All @@ -237,9 +253,9 @@ internal class CircleTileFileStorage private constructor(
while (header.hasRemaining()) channel.write(header)
val bytes = ByteBuffer.allocate(BLOCK_BYTES).order(ByteOrder.LITTLE_ENDIAN)
var widest = 0f
for (index in 0 until size) {
if (index % 4096 == 0) checkCancelled()
val item = itemAt(index)
for (position in 0 until size) {
if (position % 4096 == 0) checkCancelled()
val item = itemAt(order?.indexAt?.get(position) ?: position)
encode(bytes, bytes.position(), item)
bytes.position(bytes.position() + RECORD_BYTES)
if (item != null) widest = max(widest, CircleTilePainter.strokeWidth(item))
Expand All @@ -254,17 +270,18 @@ internal class CircleTileFileStorage private constructor(
return widest
}

/** [updates] by stored position. */
@Synchronized fun patch(
updates: List<Pair<Int, TileCircle?>>,
size: Int,
) {
check(references == 1) // Only an unpublished freshly compacted file may be written.
val bytes = ByteBuffer.allocate(RECORD_BYTES).order(ByteOrder.LITTLE_ENDIAN)
for ((index, item) in updates) {
if (index in 0 until size) {
for ((position, item) in updates) {
if (position in 0 until size) {
bytes.clear()
encode(bytes, 0, item)
var at = HEADER_BYTES + index.toLong() * RECORD_BYTES
var at = HEADER_BYTES + position.toLong() * RECORD_BYTES
while (bytes.hasRemaining()) {
val n = channel.write(bytes, at)
check(n > 0)
Expand All @@ -287,13 +304,16 @@ internal class CircleTileFileStorage private constructor(
private const val MAX_OVERRIDES = 2048
private val liveFiles = HashSet<String>()

/** [order]: the order to store them in (see the class); index order when null. */
fun create(
directory: File,
size: Int,
itemAt: (Int) -> TileCircle?,
checkCancelled: () -> Unit = {
},
order: StorageOrder? = null,
): CircleTileFileStorage {
require(order == null || order.indexAt.size == size)
require(size >= 0)
val path =
synchronized(liveFiles) {
Expand All @@ -312,8 +332,8 @@ internal class CircleTileFileStorage private constructor(
var file: SharedFile? = null
try {
file = SharedFile(path)
val widest = file.write(size, itemAt, checkCancelled)
return CircleTileFileStorage(file, size, widest)
val widest = file.write(size, itemAt, order, checkCancelled)
return CircleTileFileStorage(file, size, widest, order)
} catch (error: Throwable) {
if (file !=
null
Expand Down
90 changes: 83 additions & 7 deletions src/main/java/com/mapconductor/core/circle/CircleTileGeometry.kt
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,8 @@ internal object CircleTilePainter : ShapeTileRenderer.Painter<TileCircle> {
val item = override ?: return@forEachRecord
if (!isVisible(item)) return@forEachRecord
val slot = subpixelSlot(item, scale, originX, originY, tileSize)
if (slot >= 0) lastDot[slot] = index + 1
// The pixel keeps the circle drawn last; calls come in storage order.
if (slot >= 0 && index + 1 > lastDot[slot]) lastDot[slot] = index + 1
return@forEachRecord
}
if (bytes.getInt(p + 76) and 1 == 0) return@forEachRecord
Expand All @@ -371,16 +372,18 @@ internal object CircleTilePainter : ShapeTileRenderer.Painter<TileCircle> {
val strokeWidth = bytes.getFloat(p + 72)
if (!isVisibleColor(fill) && !(strokeWidth > 0f && isVisibleColor(stroke))) return@forEachRecord
val slot = slotOf(bytes.getDouble(p + 24), bytes.getDouble(p + 32), bytes.getDouble(p + 40), bytes.getDouble(p + 48), bytes.getDouble(p + 56))
if (slot >= 0) lastDot[slot] = index + 1
if (slot >= 0 && index + 1 > lastDot[slot]) lastDot[slot] = index + 1
}
// Gathered, then added in draw order (k): the calls come in storage order.
val deferred = DeferredCircles()
storage.forEachRecord(candidates, checkCancelled) { k, bytes, p, override ->
val index = candidates[k]
if (active != null && !active[index]) return@forEachRecord
if (bytes == null) {
val item = override ?: return@forEachRecord
val slot = subpixelSlot(item, scale, originX, originY, tileSize)
if (slot == -1 || (slot >= 0 && lastDot[slot] != index + 1)) return@forEachRecord
record(scene, item, simplifyCircle(item, scale), originX, originY, r)
deferred.addItem(k, item)
return@forEachRecord
}
if (bytes.getInt(p + 76) and 1 == 0) return@forEachRecord
Expand All @@ -404,8 +407,8 @@ internal object CircleTilePainter : ShapeTileRenderer.Painter<TileCircle> {
val ryS = (south - y) * scale
val maxR = max(rx, max(ryN, ryS))
if (maxR < CIRCLE_DOT_RADIUS_PX) {
scene.addCircle(
((floor(cx) - originX) * r).toFloat(), ((floor(cy) - originY) * r).toFloat(), r, r,
deferred.add(
k, ((floor(cx) - originX) * r).toFloat(), ((floor(cy) - originY) * r).toFloat(), r, r,
if (fillVisible) fill else stroke, 0, kind = 1,
)
return@forEachRecord
Expand All @@ -414,12 +417,14 @@ internal object CircleTilePainter : ShapeTileRenderer.Painter<TileCircle> {
if (max(abs(ryN - rx), abs(ryS - rx)) > CIRCLE_ARC_TOLERANCE_PX) throw GlShapeScene.NeedsBatches()
val outlined = strokeVisible && 2 * maxR >= MIN_OUTLINED_PX
if (!fillVisible && !outlined) return@forEachRecord
scene.addCircle(
((cx - originX) * r).toFloat(), ((cy - originY) * r).toFloat(), (rx * r).toFloat(),
deferred.add(
k, ((cx - originX) * r).toFloat(), ((cy - originY) * r).toFloat(), (rx * r).toFloat(),
if (outlined) strokeWidth * r else 0f,
if (fillVisible) fill else 0, if (outlined) stroke else 0, kind = 0,
)
}
checkCancelled()
deferred.addTo(scene) { item -> record(scene, item, simplifyCircle(item, scale), originX, originY, r) }
return true
}

Expand Down Expand Up @@ -470,3 +475,74 @@ internal object CircleTilePainter : ShapeTileRenderer.Painter<TileCircle> {
}
}
}

/**
* Circles of one tile gathered in storage order, to be added to the scene in
* draw order (`k`, their place among the tile's candidates). Primitive arrays:
* a zoomed-out tile has tens of thousands.
*/
@InternalMapConductorApi
private class DeferredCircles {
private var count = 0
private var ks = IntArray(1024)
private var floats = FloatArray(1024 * 4)
private var ints = IntArray(1024 * 3)
private val items = ArrayList<Pair<Int, TileCircle>>()

fun add(
k: Int,
x: Float,
y: Float,
a: Float,
b: Float,
fill: Int,
stroke: Int,
kind: Int,
) {
if (count == ks.size) {
ks = ks.copyOf(count * 2)
floats = floats.copyOf(count * 8)
ints = ints.copyOf(count * 6)
}
ks[count] = k
floats[count * 4] = x
floats[count * 4 + 1] = y
floats[count * 4 + 2] = a
floats[count * 4 + 3] = b
ints[count * 3] = fill
ints[count * 3 + 1] = stroke
ints[count * 3 + 2] = kind
count++
}

/** An overriding circle: recorded through the painter when its turn comes. */
fun addItem(
k: Int,
item: TileCircle,
) {
items.add(k to item)
}

/** Adds them to [scene] by k; [recordItem] records the overriding circles in their turn. */
fun addTo(
scene: GlShapeScene,
recordItem: (TileCircle) -> Unit,
) {
// k (high half) and which entry (low half; the overriding circles after the others).
val order = LongArray(count + items.size)
for (i in 0 until count) order[i] = (ks[i].toLong() shl 32) or i.toLong()
for ((j, entry) in items.withIndex()) order[count + j] = (entry.first.toLong() shl 32) or (count + j).toLong()
order.sort()
for (key in order) {
val i = key.toInt()
if (i >= count) {
recordItem(items[i - count].second)
continue
}
scene.addCircle(
floats[i * 4], floats[i * 4 + 1], floats[i * 4 + 2], floats[i * 4 + 3],
ints[i * 3], ints[i * 3 + 1], kind = ints[i * 3 + 2],
)
}
}
}
Loading
Loading