chore: Prepare Server 1.8.0

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ concurrency:
 
 jobs:
   build:
-    runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","arko"]') }}
+    runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","r730"]') }}
     steps:
       - uses: actions/checkout@v7
 

.github/workflows/coverage.yml

@@ -13,7 +13,7 @@ concurrency:
 
 jobs:
   coverage:
-    runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","arko"]') }}
+    runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","r730"]') }}
     steps:
       - uses: actions/checkout@v7
 

.github/workflows/docker.yml

@@ -21,7 +21,7 @@ concurrency:
 
 jobs:
   prepare-image:
-    runs-on: [self-hosted, Linux, X64, arko]
+    runs-on: [self-hosted, Linux, X64, r730]
     permissions:
       contents: read
     outputs:
@@ -76,7 +76,7 @@ jobs:
 
   build-platform:
     needs: prepare-image
-    runs-on: [self-hosted, Linux, X64, arko, docker]
+    runs-on: [self-hosted, Linux, X64, r730, docker]
     timeout-minutes: 20
     permissions:
       contents: read
@@ -150,7 +150,7 @@ jobs:
 
   build-and-push:
     needs: [prepare-image, build-platform]
-    runs-on: [self-hosted, Linux, X64, arko, docker]
+    runs-on: [self-hosted, Linux, X64, r730, docker]
     timeout-minutes: 10
     permissions:
       contents: read
@@ -235,7 +235,7 @@ jobs:
   notify-orchestrator:
     needs: [prepare-image, build-and-push]
     if: github.ref_name == 'dev' || github.ref_name == 'main' || startsWith(github.ref, 'refs/tags/v')
-    runs-on: [self-hosted, Linux, X64, arko]
+    runs-on: [self-hosted, Linux, X64, r730]
     permissions:
       contents: read
     env:

.github/workflows/openapi.yml

@@ -15,7 +15,7 @@ concurrency:
 
 jobs:
   validate:
-    runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","arko"]') }}
+    runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || fromJSON('["self-hosted","Linux","X64","r730"]') }}
     steps:
       - uses: actions/checkout@v7
       - name: Isolate Gradle user home

build.gradle.kts

@@ -145,8 +145,29 @@ tasks.jacocoTestCoverageVerification {
     }
 }
 
+val verifySabrBoundary = tasks.register("verifySabrBoundary") {
+    doLast {
+        val adapterRoot = file("src/main/kotlin/dev/typetype/server/sabr")
+            .canonicalFile
+            .toPath()
+        val violations = fileTree("src/main/kotlin")
+            .matching { include("**/*.kt") }
+            .files
+            .filterNot { it.canonicalFile.toPath().startsWith(adapterRoot) }
+            .flatMap { source ->
+                source.readLines().withIndex()
+                    .filter { it.value.contains("org.schabi.newpipe.extractor.services.youtube.sabr") }
+                    .map { "${source.path}:${it.index + 1}" }
+            }
+        check(violations.isEmpty()) {
+            "PipePipe SABR imports must stay in the TypeType adapter: ${violations.joinToString()}"
+        }
+    }
+}
+
 tasks.check {
     dependsOn(tasks.jacocoTestCoverageVerification)
+    dependsOn(verifySabrBoundary)
 }
 
 kotlin {

gradle.properties

@@ -1,5 +1,5 @@
 org.gradle.jvmargs=-Xmx2g -XX:+UseG1GC
 kotlin.code.style=official
-appVersion=1.7.2
+appVersion=1.8.0
 systemProp.sun.net.client.defaultReadTimeout=180000
 systemProp.sun.net.client.defaultConnectTimeout=60000

openapi.yaml

@@ -56,6 +56,7 @@ paths:
   /rss/feeds/{id}/regenerate: { $ref: ./openapi/paths/rss.yaml#/RssFeedRegenerate }
   /rss/feeds/{id}.xml: { $ref: ./openapi/paths/rss.yaml#/RssFeedDocument }
   /settings: { $ref: ./openapi/paths/access-control.yaml#/Settings }
+  /progress/batch: { $ref: ./openapi/paths/progress.yaml#/ProgressBatch }
   /backup/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeBackup }
   /restore/typetype: { $ref: ./openapi/paths/user-backup.yaml#/TypeTypeRestore }
   /portability/formats: { $ref: ./openapi/paths/portability.yaml#/PortabilityFormats }
@@ -173,6 +174,8 @@ components:
     RssFeedSecretItem: { $ref: ./openapi/components/rss.yaml#/RssFeedSecretItem }
     AdminRssFeedsPage: { $ref: ./openapi/components/rss.yaml#/AdminRssFeedsPage }
     SettingsItem: { $ref: ./openapi/components/access-control.yaml#/SettingsItem }
+    ProgressItem: { $ref: ./openapi/components/progress.yaml#/ProgressItem }
+    ProgressBatchRequest: { $ref: ./openapi/components/progress.yaml#/ProgressBatchRequest }
     TypeTypeBackupItem: { $ref: ./openapi/components/user-backup.yaml#/TypeTypeBackupItem }
     TypeTypeRestoreSummary: { $ref: ./openapi/components/user-backup.yaml#/TypeTypeRestoreSummary }
     PortabilityAdapterDescriptor: { $ref: ./openapi/components/portability.yaml#/PortabilityAdapterDescriptor }

openapi/components/progress.yaml

@@ -0,0 +1,30 @@
+ProgressItem:
+  type: object
+  required: [videoUrl, position, updatedAt]
+  properties:
+    videoUrl:
+      type: string
+      format: uri
+    position:
+      type: integer
+      format: int64
+      minimum: 0
+      description: Playback position in milliseconds.
+    updatedAt:
+      type: integer
+      format: int64
+      minimum: 0
+      description: Unix timestamp in milliseconds.
+
+ProgressBatchRequest:
+  type: object
+  required: [videoUrls]
+  properties:
+    videoUrls:
+      type: array
+      minItems: 1
+      maxItems: 200
+      items:
+        type: string
+        format: uri
+        maxLength: 2048

openapi/paths/portability.yaml

@@ -16,6 +16,11 @@ PortabilityImports:
   post:
     tags: [portability]
     summary: Upload and analyze an account backup
+    description: |
+      Accepts one backup of at most 512 MiB. For large YouTube Takeout exports,
+      clients can package only the YouTube CSV, HTML and JSON metadata, preserving
+      entry paths and excluding uploaded media and unrelated Google products.
+      The server still validates the archive and parses all imported records.
     parameters:
       - name: format
         in: query
@@ -39,7 +44,8 @@ PortabilityImports:
             schema: { $ref: ../components/portability.yaml#/PortabilityJobSnapshot }
       '400': { description: Invalid or unrecognized backup }
       '401': { description: Missing or invalid token }
-      '413': { description: Upload exceeds the configured limit }
+      '413':
+        description: Upload exceeds 512 MiB (portability_upload_too_large)
 PortabilityExports:
   post:
     tags: [portability]

openapi/paths/progress.yaml

@@ -0,0 +1,33 @@
+ProgressBatch:
+  post:
+    tags: [user-data]
+    summary: Read playback progress for multiple videos
+    security:
+      - bearerAuth: []
+    requestBody:
+      required: true
+      content:
+        application/json:
+          schema:
+            $ref: ../components/progress.yaml#/ProgressBatchRequest
+    responses:
+      '200':
+        description: Progress entries in request order. Videos without saved progress have position and updatedAt set to zero.
+        content:
+          application/json:
+            schema:
+              type: array
+              items:
+                $ref: ../components/progress.yaml#/ProgressItem
+      '400':
+        description: Invalid request body or video URL list.
+        content:
+          application/json:
+            schema:
+              $ref: ../components/common.yaml#/ErrorResponse
+      '401':
+        description: Authentication required.
+        content:
+          application/json:
+            schema:
+              $ref: ../components/common.yaml#/ErrorResponse

src/main/kotlin/dev/typetype/server/AppMetrics.kt

@@ -43,6 +43,7 @@ fun metricPath(path: String): String = when {
     path.startsWith("/downloader/jobs/") && path.endsWith("/artifact") -> "/downloader/jobs/{id}/artifact"
     path.startsWith("/downloader/jobs/") && path.endsWith("/cancel") -> "/downloader/jobs/{id}/cancel"
     path.startsWith("/downloader/jobs/") -> "/downloader/jobs/{id}"
+    path == "/progress/batch" -> path
     path.startsWith("/progress/") -> "/progress/{videoUrl}"
     path.startsWith("/favorites/") -> "/favorites/{videoUrl}"
     path.startsWith("/watch-later/") -> "/watch-later/{videoUrl}"

src/main/kotlin/dev/typetype/server/Plugins.kt

@@ -1,6 +1,8 @@
 package dev.typetype.server
 
 import dev.typetype.server.models.ErrorResponse
+import dev.typetype.server.routes.isMultipartSizeLimit
+import dev.typetype.server.routes.respondPortabilityError
 import dev.typetype.server.services.AuthService
 import io.ktor.http.HttpHeaders
 import io.ktor.http.HttpMethod
@@ -118,6 +120,11 @@ internal fun Application.configureStatusPages() {
         exception<Throwable> { call, cause ->
             if (cause is io.ktor.utils.io.ClosedWriteChannelException) return@exception
             if (cause is kotlinx.coroutines.CancellationException) throw cause
+            // Ktor's multipart producer can fail outside the route's receive block.
+            if (call.request.path() == "/portability/imports" && cause.isMultipartSizeLimit()) {
+                call.respondPortabilityError(dev.typetype.server.portability.PortabilityUploadTooLargeException())
+                return@exception
+            }
             log.error("Unhandled exception requestId=${call.requestId()} path=${call.request.path()}", cause)
             call.respond(HttpStatusCode.InternalServerError, ErrorResponse("Internal server error", "internal_error"))
         }

src/main/kotlin/dev/typetype/server/routes/PortabilityMultipartFailure.kt

@@ -0,0 +1,10 @@
+package dev.typetype.server.routes
+
+import java.io.IOException
+
+internal fun Throwable.isMultipartSizeLimit(): Boolean = this is IOException && (
+    message?.let {
+        (it.startsWith("Multipart content length exceeds limit ") && "formFieldLimit" in it) ||
+            (it.startsWith("Limit of ") && " bytes exceeded while searching for " in it)
+    } == true
+)

src/main/kotlin/dev/typetype/server/routes/PortabilityRouteSupport.kt

@@ -17,6 +17,11 @@ internal fun parsePortabilityFormat(value: String?): PortabilityFormat? {
 }
 
 internal suspend fun ApplicationCall.respondPortabilityError(error: Exception) {
+    if (error is kotlinx.coroutines.CancellationException) throw error
+    if (error.isMultipartSizeLimit()) {
+        respondPortabilityError(PortabilityUploadTooLargeException())
+        return
+    }
     val status = when (error) {
         is PortabilityJobNotFoundException -> HttpStatusCode.NotFound
         is PortabilityUploadTooLargeException -> HttpStatusCode.PayloadTooLarge

src/main/kotlin/dev/typetype/server/routes/ProgressRoutes.kt

@@ -10,13 +10,29 @@ import io.ktor.server.request.receive
 import io.ktor.server.response.respond
 import io.ktor.server.routing.Route
 import io.ktor.server.routing.get
+import io.ktor.server.routing.post
 import io.ktor.server.routing.put
 import kotlinx.serialization.Serializable
 
 @Serializable
 internal data class ProgressBody(val position: Long)
 
+@Serializable
+internal data class ProgressBatchBody(val videoUrls: List<String>)
+
 fun Route.progressRoutes(progressService: ProgressService, authService: AuthService, settingsService: SettingsService? = null) {
+    post("/progress/batch") {
+        call.withJwtAuth(authService) { userId ->
+            val body = runCatching { call.receive<ProgressBatchBody>() }.getOrElse {
+                return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid request body"))
+            }
+            if (body.videoUrls.isEmpty() || body.videoUrls.size > MAX_PROGRESS_BATCH_SIZE || body.videoUrls.any { it.isBlank() || it.length > MAX_VIDEO_URL_LENGTH }) {
+                return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Invalid videoUrls"))
+            }
+            val videoUrls = body.videoUrls.distinct()
+            call.respond(progressService.getMany(userId, videoUrls))
+        }
+    }
     get("/progress/{videoUrl...}") {
         call.withJwtAuth(authService) { userId ->
             val videoUrl = call.urlTailParameter("videoUrl") ?: return@withJwtAuth call.respond(HttpStatusCode.BadRequest, ErrorResponse("Missing videoUrl"))
@@ -59,6 +75,9 @@ fun Route.progressRoutes(progressService: ProgressService, authService: AuthServ
     }
 }
 
+private const val MAX_PROGRESS_BATCH_SIZE = 200
+private const val MAX_VIDEO_URL_LENGTH = 2_048
+
 private fun skippedProgress(videoUrl: String, position: Long): ProgressItem = ProgressItem(
     videoUrl = videoUrl,
     position = position.coerceAtLeast(0L),

src/main/kotlin/dev/typetype/server/routes/SabrFormatSelector.kt

@@ -1,7 +1,7 @@
 package dev.typetype.server.routes
 
-import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat
-import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrInfo
+import dev.typetype.server.sabr.YoutubeSabrFormat
+import dev.typetype.server.sabr.YoutubeSabrInfo
 
 internal object SabrFormatSelector {
     fun video(info: YoutubeSabrInfo, itag: Int?): YoutubeSabrFormat? {
@@ -23,7 +23,7 @@ internal object SabrFormatSelector {
     }
 
     private fun YoutubeSabrFormat.matchesAudio(itag: Int?, trackId: String?, requireAac: Boolean): Boolean =
-        itag != null && isAudio && getItag() == itag && (!requireAac || isAac()) &&
+        itag != null && isAudio && this.itag == itag && (!requireAac || isAac()) &&
             (trackId.isNullOrBlank() || audioTrackId == trackId)
 
     private fun YoutubeSabrFormat.isAac(): Boolean =

src/main/kotlin/dev/typetype/server/routes/SabrManifestHandler.kt

@@ -13,7 +13,7 @@ import dev.typetype.server.services.SabrSessionPurpose
 import dev.typetype.server.services.SabrSessionStore
 import dev.typetype.server.services.StreamService
 import dev.typetype.server.services.bothFormatsKnown
-import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat
+import dev.typetype.server.sabr.YoutubeSabrFormat
 import io.ktor.http.HttpStatusCode
 import io.ktor.server.application.ApplicationCall
 import io.ktor.server.response.respond

src/main/kotlin/dev/typetype/server/routes/SabrManifestResponse.kt

@@ -58,7 +58,7 @@ internal suspend fun ApplicationCall.respondSabrManifest(
     respondText(manifest, if (hls) HLS_CONTENT_TYPE else DASH_CONTENT_TYPE)
 }
 
-private fun SabrSessionHolder.startSegment(format: org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat): Int =
+private fun SabrSessionHolder.startSegment(format: dev.typetype.server.sabr.YoutubeSabrFormat): Int =
     key.startTimeMs.takeIf { it > 0L }
         ?.let { playbackStartSequence(format, it) }
         ?: 1

src/main/kotlin/dev/typetype/server/routes/SabrPlaybackHandler.kt

@@ -19,7 +19,7 @@ import io.ktor.http.HttpStatusCode
 import io.ktor.server.application.ApplicationCall
 import io.ktor.server.request.receive
 import io.ktor.server.response.respond
-import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat
+import dev.typetype.server.sabr.YoutubeSabrFormat
 
 internal class SabrPlaybackHandler(
     private val sabrSessionStore: SabrSessionStore,

src/main/kotlin/dev/typetype/server/routes/SabrPlaybackModels.kt

@@ -1,7 +1,7 @@
 package dev.typetype.server.routes
 
 import kotlinx.serialization.Serializable
-import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest
+import dev.typetype.server.sabr.SabrSegmentRequest
 
 @Serializable
 internal data class SabrPlaybackRequest(

src/main/kotlin/dev/typetype/server/routes/SabrPlaybackStateHandler.kt

@@ -8,7 +8,7 @@ import dev.typetype.server.services.livePlaybackSnapshot
 import io.ktor.http.HttpStatusCode
 import io.ktor.server.application.ApplicationCall
 import io.ktor.server.response.respond
-import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest
+import dev.typetype.server.sabr.SabrSegmentRequest
 
 internal class SabrPlaybackStateHandler(private val sabrSessionStore: SabrSessionStore) {
     suspend fun get(call: ApplicationCall, sessionId: String) {

src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowBuilder.kt

@@ -12,8 +12,8 @@ import dev.typetype.server.services.livePlaybackSnapshot
 import dev.typetype.server.services.playbackContinuationSequence
 import dev.typetype.server.services.playbackSegmentStartMs
 import dev.typetype.server.services.resolvePlaybackStartMs
-import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest
-import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat
+import dev.typetype.server.sabr.SabrSegmentRequest
+import dev.typetype.server.sabr.YoutubeSabrFormat
 
 private const val MAX_SEGMENTS_PER_TRACK = 12
 

src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowHandler.kt

@@ -12,7 +12,7 @@ import io.ktor.http.HttpStatusCode
 import io.ktor.server.application.ApplicationCall
 import io.ktor.server.request.receive
 import io.ktor.server.response.respond
-import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest
+import dev.typetype.server.sabr.SabrSegmentRequest
 
 internal class SabrPlaybackWindowHandler(private val sabrSessionStore: SabrSessionStore) {
     private val windowBuilder = SabrPlaybackWindowBuilder(sabrSessionStore)

src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowSegmentTiming.kt

@@ -4,8 +4,8 @@ import dev.typetype.server.services.CachedSabrSegment
 import dev.typetype.server.services.SabrSessionHolder
 import dev.typetype.server.services.SabrSessionStore
 import dev.typetype.server.services.playbackSegmentDurationMs
-import org.schabi.newpipe.extractor.services.youtube.sabr.SabrSegmentRequest
-import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat
+import dev.typetype.server.sabr.SabrSegmentRequest
+import dev.typetype.server.sabr.YoutubeSabrFormat
 
 internal suspend fun SabrSessionStore.resolvePlaybackDurationMs(
     holder: SabrSessionHolder,

src/main/kotlin/dev/typetype/server/routes/SabrPlaybackWindowTiming.kt

@@ -4,7 +4,7 @@ import dev.typetype.server.services.CachedSabrSegment
 import dev.typetype.server.services.SabrSessionHolder
 import dev.typetype.server.services.livePlaybackSnapshot
 import dev.typetype.server.services.playbackSegmentDurationMs
-import org.schabi.newpipe.extractor.services.youtube.sabr.YoutubeSabrFormat
+import dev.typetype.server.sabr.YoutubeSabrFormat
 
 internal fun SabrSessionHolder.durationMs(): Long {
     livePlaybackSnapshot()?.let { live ->