diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b219a92..199231a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,11 +5,9 @@ # Category C (tools-version 6.3): macos-15 + 6.3, macos-26 + 6.3 # Linux: ubuntu + Swift 6.3 # -# This package is Category A, plus a Linux leg on Swift 6.3. Test names are raw +# This package is Category A, plus a Linux leg. Test names are raw # identifiers (SE-0451), so a leg must be on Swift 6.2 or newer to run `swift test`; # a leg on an older compiler is limited to `swift build -v`. -# -# When Swift 6.4 ships: add 6.4 legs alongside 6.2 name: CI @@ -32,8 +30,12 @@ jobs: include: - os: macos-26 swift: "6.2" + - os: macos-26 + swift: "6.4" - os: ubuntu-latest swift: "6.3" + - os: ubuntu-latest + swift: "6.4" runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/dof-cycle-watch.yml b/.github/workflows/dof-cycle-watch.yml new file mode 100644 index 0000000..a0fbe80 --- /dev/null +++ b/.github/workflows/dof-cycle-watch.yml @@ -0,0 +1,480 @@ +name: DOF Cycle Watch + +# The FAA releases each 56-day DOF the day after it takes effect — the published schedule runs +# "reflects changes to" August 02, released August 04, for the cycle effective August 03 — so +# unlike NASR there is no window in which the incoming cycle can be parsed early. The watch +# therefore tracks the cycle already in effect. A cheap probe gates the expensive parse; the +# report artifact, named for the cycle, doubles as the record of what has already been checked. + +on: + schedule: + - cron: "0 16 * * *" + workflow_dispatch: + inputs: + cycle: + description: "Cycle to check, as the YYYY-MM-DD currency date in its filename. Blank uses the cycle in effect." + required: false + type: string + +permissions: {} + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + probe: + name: Look for a new cycle + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read + issues: write + outputs: + cycle: ${{ steps.target.outputs.cycle }} + cycle-id: ${{ steps.target.outputs.cycle-id }} + previous-cycle: ${{ steps.target.outputs.previous-cycle }} + url: ${{ steps.availability.outputs.url }} + parse: ${{ steps.decide.outputs.parse }} + steps: + - name: Resolve the target cycle + id: target + env: + REQUESTED: ${{ inputs.cycle }} + run: | + set -euo pipefail + + # Cycles take effect every 56 days from 2025-09-01. The FAA names the file for the last + # day of coverage, the day before, which is also the currency date in its header — so + # that is the identity used throughout, and it is what the report has to agree with. + period=$(( 56 * 86400 )) + datum=$(date -u -d 2025-09-01 +%s) + index=$(( ($(date -u +%s) - datum) / period )) + currency=$(( datum + index * period - 86400 )) + + if [ -n "$REQUESTED" ]; then + if ! requested=$(date -u -d "$REQUESTED" +%s 2>/dev/null); then + echo "::error::'$REQUESTED' is not a date. Use YYYY-MM-DD." + exit 1 + fi + if [ $(( (requested - (datum - 86400)) % period )) -ne 0 ]; then + echo "::error::$REQUESTED is not a DOF currency date; they fall every 56 days from 2025-08-31." + exit 1 + fi + currency=$requested + fi + + { + echo "cycle=$(date -u -d "@$currency" +%F)" + echo "cycle-id=$(date -u -d "@$currency" +%Y%m%d)" + echo "previous-cycle=$(date -u -d "@$(( currency - period ))" +%F)" + echo "effective=$(date -u -d "@$(( currency + 86400 ))" +%F)" + } >> "$GITHUB_OUTPUT" + + - name: Check whether the FAA has published it + id: availability + env: + CYCLE: ${{ steps.target.outputs.cycle }} + EFFECTIVE: ${{ steps.target.outputs.effective }} + run: | + set -euo pipefail + + # aeronav.faa.gov answers HEAD honestly, so no range-GET workaround is needed here. + url="https://aeronav.faa.gov/Obst_Data/DOF_$(date -u -d "$CYCLE" +%y%m%d).zip" + code=$(curl --silent --show-error --location --head \ + --retry 3 --retry-delay 10 --retry-all-errors \ + --dump-header headers.txt --output /dev/null \ + --write-out '%{http_code}' "$url" || echo 000) + echo "HEAD $url -> $code" + + if [ "$code" = "200" ]; then + available=true + else + available=false + fi + + # Last-Modified is the mirror's re-sync time, not the release date: the file released on + # 2026-08-04 reported a Last-Modified of 2026-09-14. It is recorded, never relied on. + modified=$(grep -i '^last-modified:' headers.txt | tail -n 1 | cut -d' ' -f2- | tr -d '\r') + size=$(grep -i '^content-length:' headers.txt | tail -n 1 | cut -d' ' -f2- | tr -d '\r') + days_since=$(( ( $(date -u +%s) - $(date -u -d "$EFFECTIVE" +%s) ) / 86400 )) + + # Release lands one day after the cycle takes effect. A week of silence means the FAA + # slipped or the URL shape changed, either of which looks like an ordinary quiet day. + if [ "$available" != "true" ] && [ "$days_since" -ge 7 ]; then + overdue=true + else + overdue=false + fi + + { + echo "available=$available" + echo "overdue=$overdue" + echo "days-since=$days_since" + echo "last-modified=$modified" + echo "size=$size" + echo "code=$code" + echo "url=$url" + } >> "$GITHUB_OUTPUT" + + - name: Decide whether to parse + id: decide + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + CYCLE: ${{ steps.target.outputs.cycle }} + AVAILABLE: ${{ steps.availability.outputs.available }} + DAYS_SINCE: ${{ steps.availability.outputs.days-since }} + SIZE: ${{ steps.availability.outputs.size }} + REQUESTED: ${{ inputs.cycle }} + ARTIFACT: dof-report-${{ steps.target.outputs.cycle }} + run: | + set -euo pipefail + + note() { echo "$1"; echo "$1" >> "$GITHUB_STEP_SUMMARY"; } + + if [ "$AVAILABLE" != "true" ]; then + note "Cycle $CYCLE is not downloadable yet; it took effect $DAYS_SINCE day(s) ago." + echo "parse=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ -n "$REQUESTED" ]; then + note "Re-checking cycle $CYCLE on request." + echo "parse=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # An existing report artifact is the record that this cycle was already checked. + found=$(gh api "repos/$GH_REPO/actions/artifacts?name=$ARTIFACT&per_page=100" \ + --jq '[.artifacts[] | select(.name == env.ARTIFACT and .expired == false)] | length') + + if [ "$found" -gt 0 ]; then + note "Cycle $CYCLE has already been checked." + echo "parse=false" >> "$GITHUB_OUTPUT" + else + note "Cycle $CYCLE is new ($SIZE bytes, in effect $DAYS_SINCE day(s))." + echo "parse=true" >> "$GITHUB_OUTPUT" + fi + + - name: Report a cycle the FAA never published + if: steps.availability.outputs.overdue == 'true' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + CYCLE: ${{ steps.target.outputs.cycle }} + EFFECTIVE: ${{ steps.target.outputs.effective }} + DAYS_SINCE: ${{ steps.availability.outputs.days-since }} + URL: ${{ steps.availability.outputs.url }} + CODE: ${{ steps.availability.outputs.code }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + + { + echo "Cycle **$CYCLE** took effect on $EFFECTIVE, $DAYS_SINCE day(s) ago, and is still" + echo "not downloadable. The FAA normally releases the day after a cycle takes effect," + echo "so either this cycle slipped or the download URL has changed shape." + echo + echo "| URL | HEAD |" + echo "|---|---|" + echo "| \`$URL\` | $CODE |" + echo + echo "[View the run]($RUN_URL)" + } > issue-body.md + + title="DOF cycle $CYCLE was never published" + number=$(TITLE="$title" gh issue list --state all --search "DOF cycle $CYCLE in:title" \ + --json number,title --jq 'map(select(.title == env.TITLE)) | .[0].number // empty') + + if [ -n "$number" ]; then + gh issue comment "$number" --body-file issue-body.md + else + gh issue create --title "$title" --body-file issue-body.md + fi + + parse: + name: Parse the cycle + needs: probe + if: needs.probe.outputs.parse == 'true' + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + actions: read + issues: write + env: + CYCLE: ${{ needs.probe.outputs.cycle }} + CYCLE_ID: ${{ needs.probe.outputs.cycle-id }} + PREVIOUS_CYCLE: ${{ needs.probe.outputs.previous-cycle }} + URL: ${{ needs.probe.outputs.url }} + steps: + - uses: actions/checkout@v7 + + - name: Name the report and the log + run: | + set -euo pipefail + { + echo "REPORT=$RUNNER_TEMP/report.json" + echo "PARSE_LOG=$RUNNER_TEMP/parse.log" + } >> "$GITHUB_ENV" + + - uses: SwiftyLab/setup-swift@latest + with: + swift-version: "6.4" + + - name: Cache SwiftPM build products + # Usually misses: 56 days between runs is well past the seven-day idle eviction. It earns + # its place when a cycle is re-checked by hand after a fix. + uses: actions/cache@v6 + with: + path: | + .build + ~/.cache/org.swift.swiftpm + key: swiftpm-cyclewatch-ubuntu-latest-swift6.4-${{ hashFiles('Package.resolved') }} + + - name: Build + # Release: a debug build cannot get through 650,000 fixed-width records in any useful time. + run: swift build -c release --product SwiftDOF_E2E + + - name: Fetch the previous cycle's report + id: baseline + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + ARTIFACT: dof-report-${{ needs.probe.outputs.previous-cycle }} + run: | + set -euo pipefail + + # download-artifact cannot reach another run's artifact without that run's id, which is + # itself state to keep somewhere; the name already carries the cycle. + id=$(gh api "repos/$GH_REPO/actions/artifacts?name=$ARTIFACT&per_page=100" \ + --jq '[.artifacts[] | select(.name == env.ARTIFACT and .expired == false)] + | max_by(.id) | .id // empty') + + if [ -z "$id" ]; then + echo "No report for $PREVIOUS_CYCLE; this cycle becomes the baseline for the next one." + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + gh api "repos/$GH_REPO/actions/artifacts/$id/zip" > previous.zip + unzip -o -q previous.zip -d previous + + # A run that crashed before writing its report still uploaded its log, so the artifact + # existing does not mean there is anything to compare against. + if [ -f previous/report.json ]; then + echo "found=true" >> "$GITHUB_OUTPUT" + else + echo "The report for $PREVIOUS_CYCLE holds no counts; the comparison is skipped." + echo "found=false" >> "$GITHUB_OUTPUT" + fi + + - name: Parse the cycle + id: parse + env: + BASELINE_FOUND: ${{ steps.baseline.outputs.found }} + run: | + set -euo pipefail + + args=(--input "$URL" --report "$REPORT") + if [ "$BASELINE_FOUND" = "true" ]; then + args+=(--baseline previous/report.json) + fi + + # The exit code is captured, not propagated: the report still has to be uploaded, + # summarized, and turned into an issue. + status=0 + .build/release/SwiftDOF_E2E "${args[@]}" 2>&1 | tee "$PARSE_LOG" || status=$? + echo "status=$status" >> "$GITHUB_OUTPUT" + + - name: Upload the report + # Uploaded before anything reads it, and on failure too: this artifact is also the record + # that the cycle was checked, so a broken cycle is reported once rather than every day. + if: always() + uses: actions/upload-artifact@v7 + with: + name: dof-report-${{ needs.probe.outputs.cycle }} + path: | + ${{ runner.temp }}/report.json + ${{ runner.temp }}/parse.log + retention-days: 90 + if-no-files-found: warn + + - name: Read the verdict out of the report + id: verdict + if: always() + run: | + set -euo pipefail + + failed=unknown + drift=0 + declared= + if [ -f "$REPORT" ]; then + failed=$(jq -r '.failed' "$REPORT") + drift=$(jq '.drift | length' "$REPORT") + declared=$(jq -r '.cycle' "$REPORT") + fi + + # The header's currency date is computed independently of this workflow's arithmetic, so + # a disagreement means one of the two is wrong about which cycle this file even is. + if [ -n "$declared" ] && [ "$declared" != "$CYCLE_ID" ]; then + mismatch=true + echo "::error::The file at $URL declares cycle $declared, not $CYCLE_ID." + else + mismatch=false + fi + + { + echo "failed=$failed" + echo "drift=$drift" + echo "declared=$declared" + echo "mismatch=$mismatch" + } >> "$GITHUB_OUTPUT" + + - name: Write the job summary + if: always() + env: + DECLARED: ${{ steps.verdict.outputs.declared }} + run: | + set -euo pipefail + + if [ ! -f "$REPORT" ]; then + echo "The run produced no report for cycle \`$CYCLE\`; see the log." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + { + echo "## DOF cycle $CYCLE" + echo + echo "- Source: \`$URL\`" + echo "- Cycle the file declares: \`$DECLARED\`" + echo "- Obstacles parsed: $(jq -r '.obstacleCount' "$REPORT")" + echo "- Parse errors: $(jq -r '.parseErrorCount' "$REPORT")" + echo "- Outcome: $(jq -r 'if .failed then "**failed**" else "clean" end' "$REPORT")" + jq -r '.failureReasons[]? | " - \(.)"' "$REPORT" + echo + + if [ "$(jq '.drift | length' "$REPORT")" -gt 0 ]; then + echo "### Count drift against $PREVIOUS_CYCLE" + echo + echo "| Region | $PREVIOUS_CYCLE | $CYCLE | Reason |" + echo "|---|---:|---:|---|" + jq -r '.drift[] | "| \(.scope) | \(.was) | \(.now // "missing") | \(.reason) |"' "$REPORT" + echo + fi + + if [ "$(jq '.errorSamples | length' "$REPORT")" -gt 0 ]; then + echo "### Error samples" + echo + echo '```' + jq -r '.errorSamples[] | "line \(.line): \(.reason // .message)"' "$REPORT" + echo '```' + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Open or update the cycle's issue + # Drift files an issue as readily as a parse failure: a region that quietly empties produces + # no errors at all, so the count diff is the only thing that would ever mention it. + if: >- + always() && steps.parse.outcome != 'skipped' + && (steps.parse.outputs.status != '0' || steps.verdict.outputs.drift != '0' + || steps.verdict.outputs.mismatch == 'true') + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + STATUS: ${{ steps.parse.outputs.status }} + DECLARED: ${{ steps.verdict.outputs.declared }} + MISMATCH: ${{ steps.verdict.outputs.mismatch }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + + # Everything from the report reaches the body through a file, so nothing the FAA ships is + # ever read as shell. + { + echo "The end-to-end check for cycle **$CYCLE** found problems." + echo + echo "- Source: \`$URL\`" + echo "[View the run]($RUN_URL) — the \`dof-report-$CYCLE\` artifact holds the full" + echo "report and the tool's log." + echo + + if [ "$MISMATCH" = "true" ]; then + echo "### Wrong cycle" + echo + echo "The file declares currency date \`$DECLARED\`, but this URL should hold" + echo "\`$CYCLE_ID\`. Either the FAA published the wrong file or the cycle arithmetic" + echo "in \`Cycle\` and in this workflow has drifted apart." + echo + fi + + if [ -f "$REPORT" ]; then + if [ "${STATUS:-1}" != "0" ]; then + echo "### Failures" + echo + jq -r '.failureReasons[]? | "- \(.)"' "$REPORT" + echo + fi + + if [ "$(jq '.drift | length' "$REPORT")" -gt 0 ]; then + echo "### Count drift against $PREVIOUS_CYCLE" + echo + echo "| Region | $PREVIOUS_CYCLE | $CYCLE | Reason |" + echo "|---|---:|---:|---|" + jq -r '.drift[] | "| \(.scope) | \(.was) | \(.now // "missing") | \(.reason) |"' "$REPORT" + echo + fi + + echo "### Counts" + echo + echo "- Obstacles parsed: $(jq -r '.obstacleCount' "$REPORT")" + echo "- Parse errors: $(jq -r '.parseErrorCount' "$REPORT")" + echo + + if [ "$(jq '.errorSamples | length' "$REPORT")" -gt 0 ]; then + echo "### Error samples" + echo + echo '```' + jq -r '.errorSamples[] | "line \(.line): \(.reason // .message)"' "$REPORT" + echo '```' + fi + else + echo "The run produced no report. The tail of its log:" + echo + echo '```' + tail -c 8000 "$PARSE_LOG" 2>/dev/null || echo "No log was produced either." + echo '```' + fi + } > issue-body.md + + # The title is a function of the cycle alone, which is what keeps a re-check idempotent. + # The search omits the punctuation so GitHub's query parser cannot read it as a qualifier. + title="DOF cycle $CYCLE: end-to-end check failed" + number=$(TITLE="$title" gh issue list --state all --search "DOF cycle $CYCLE in:title" \ + --json number,title --jq 'map(select(.title == env.TITLE)) | .[0].number // empty') + + if [ -n "$number" ]; then + gh issue reopen "$number" >/dev/null 2>&1 || true + gh issue comment "$number" --body-file issue-body.md + else + gh issue create --title "$title" --body-file issue-body.md + fi + + - name: Annotate the outcome + if: always() && steps.parse.outcome != 'skipped' + env: + STATUS: ${{ steps.parse.outputs.status }} + DRIFT: ${{ steps.verdict.outputs.drift }} + run: | + set -euo pipefail + + drift=${DRIFT:-0} + + if [ "${STATUS:-1}" != "0" ]; then + echo "::error::Cycle $CYCLE failed its end-to-end check; an issue has been filed." + elif [ "$drift" != "0" ]; then + echo "::warning::Cycle $CYCLE parsed cleanly but $drift count(s) drifted." + else + echo "Cycle $CYCLE parsed cleanly with no count drift." + fi diff --git a/.periphery.yml b/.periphery.yml index 85b884a..b5d9e03 100644 --- a/.periphery.yml +++ b/.periphery.yml @@ -1 +1,4 @@ retain_public: true +# The report the cycle-watch workflow reads is written by JSONEncoder and read by jq, so its +# properties have no Swift reader for Periphery to find. +retain_codable_properties: true diff --git a/CHANGELOG.md b/CHANGELOG.md index bd53185..efb2b59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,50 @@ # Change Log +## [Unreleased] + +### Added + +- `Obstacle.verticalAccuracy`, the accuracy category of an obstacle's reported + height, read from column 100 of the DOF record. +- A `DOF Cycle Watch` workflow that parses each 56-day cycle as the FAA releases + it and opens an issue when a cycle fails to parse, when counts drift from the + previous cycle, or when the file declares a currency date other than the one + its URL should hold. +- `SwiftDOF_E2E --report` writes a JSON report of a parse — counts, per-region + totals, and the field each failed line failed on — and `--baseline` compares + those counts against an earlier report. The tool now exits non-zero when any + line fails to parse. + +### Changed + +- **Breaking.** `AccuracyCategory` is replaced by two enums that each carry only + the codes the FAA defines for their column: `HorizontalAccuracy` (`1`-`9`) and + `VerticalAccuracy` (`A`-`I`). `Obstacle.horizontalAccuracy` is now a + `HorizontalAccuracy`. The `.survey` case (±3 feet) becomes + `VerticalAccuracy.codeA`, the vertical column it belongs to. Cases are named + for the code the FAA publishes — `.code1`, `.codeA` — rather than a "category" + the criteria never use, and the tolerance each one guarantees is now spelled + `tolerance` rather than `accuracy`. +- **Breaking.** `MarkingType`'s cases are the mark indicator codes the DOF + actually uses — `orangeOrOrangeWhitePaint` (`P`), `whitePaintOnly` (`W`), + `marked` (`M`), `flagMarker` (`F`), `sphericalMarker` (`S`), `none` (`N`), and + `unknown` (`U`) — in place of the `A`-`I` cases, which described a code set + that appears nowhere in the DOF. A blank column parses as `.unknown`. + +- Adopt typed throws across the parsing surface: `DOF.init(data:)`, `DOF.init(url:)`, the `DOF.from(…)` factories, and `DOFByteParser` now declare `throws(DOFError)`, and the DOF file line reader's `AsyncIteratorProtocol.Failure` is `DOFError`. `AsyncBytesLineReader` propagates its source sequence's own `Failure` type. +- `DOF.from(filePath:)` streams the file in chunks instead of reading it into memory in its entirety. A file that cannot be opened now throws `DOFError.fileNotFound` rather than a Foundation file-read error. +- Match the header's "CURRENCY DATE = " marker against an `InlineArray<16, UInt8>`, removing a heap allocation from currency date parsing. + +### Fixed + +- `Obstacle.marking` reads the mark indicator from column 102 rather than column + 100, which holds the vertical accuracy code. Because the old `MarkingType` + cases `A`-`I` coincided exactly with the vertical accuracy code set, every + record in every cycle parsed without error and reported the wrong marking — + an obstacle with a ±50 foot height tolerance (`D`) read as `.paintAndFlags`. + The vertical accuracy column is now exposed as `Obstacle.verticalAccuracy`. +- `Cycle.previous`, `Cycle.next`, and the cycle datum date no longer force-unwrap optionals. + ## [1.3.0] - 2026-09-14 ### Changed diff --git a/Package.swift b/Package.swift index 71bf0e7..fbab8a3 100644 --- a/Package.swift +++ b/Package.swift @@ -9,13 +9,14 @@ let upcomingFeatures: [SwiftSetting] = [ .enableUpcomingFeature("ImmutableWeakCaptures"), .enableUpcomingFeature("MemberImportVisibility"), .enableUpcomingFeature("ExistentialAny"), - .enableUpcomingFeature("InternalImportsByDefault") + .enableUpcomingFeature("InternalImportsByDefault"), + .strictMemorySafety() ] let package = Package( name: "SwiftDOF", defaultLocalization: "en", - platforms: [.macOS(.v15), .iOS(.v18), .watchOS(.v11), .tvOS(.v18), .visionOS(.v2)], + platforms: [.macOS(.v26), .iOS(.v26), .watchOS(.v26), .tvOS(.v26), .visionOS(.v26)], products: [ // Products define the executables and libraries a package produces, making them visible to other packages. .library( diff --git a/README.md b/README.md index 2f4b108..7d940ca 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ The DOF format is documented at `: Path or URL to DOF file (.dat or .zip). Defaults to current FAA cycle. - `-f, --format `: Output format. Defaults to summary. +- `--report `: Write a JSON report of the parse — obstacle count, parse + errors and the field each one failed on, and per-region counts. +- `--baseline `: Compare counts against an earlier report and record any + that moved more than their tolerance. + +The tool exits non-zero when any line fails to parse. + +### Cycle Watch + +The `DOF Cycle Watch` workflow parses each new 56-day cycle as the FAA releases +it. A daily probe checks whether the cycle in effect is downloadable yet, and +the first time it is, the cycle is parsed end to end and its report kept as an +artifact — both the record that the cycle was checked and the baseline the next +cycle's counts are compared against. Parse errors, counts that drift, or a file +declaring the wrong currency date each open an issue naming the cycle. + +To re-check a cycle by hand, run the workflow and give it the currency date from +the filename, as `YYYY-MM-DD`. diff --git a/Sources/SwiftDOF/Cycle.swift b/Sources/SwiftDOF/Cycle.swift index e087ea9..c5013e6 100644 --- a/Sources/SwiftDOF/Cycle.swift +++ b/Sources/SwiftDOF/Cycle.swift @@ -26,7 +26,10 @@ public struct Cycle: Sendable, Codable, Equatable, Hashable { month: datum.month, day: datum.day ) - return calendar.date(from: components)! + guard let date = calendar.date(from: components) else { + preconditionFailure("The DOF datum is not a valid Gregorian date") + } + return date } /// The currently effective cycle based on today's date. @@ -42,34 +45,10 @@ public struct Cycle: Sendable, Codable, Equatable, Hashable { public let day: UInt8 /// The cycle preceding this one (56 days earlier). - public var previous: Self? { - guard let firstDate, - let previousDate = Self.calendar.date(byAdding: .day, value: -Self.period, to: firstDate) - else { - return nil - } - let components = Self.calendar.dateComponents([.year, .month, .day], from: previousDate) - return Self( - year: UInt(components.year!), - month: UInt8(components.month!), - day: UInt8(components.day!) - ) - } + public var previous: Self? { cycle(offsetByDays: -Self.period) } /// The cycle following this one (56 days later). - public var next: Self? { - guard let firstDate, - let nextDate = Self.calendar.date(byAdding: .day, value: Self.period, to: firstDate) - else { - return nil - } - let components = Self.calendar.dateComponents([.year, .month, .day], from: nextDate) - return Self( - year: UInt(components.year!), - month: UInt8(components.month!), - day: UInt8(components.day!) - ) - } + public var next: Self? { cycle(offsetByDays: Self.period) } /// Whether this cycle falls on a valid cycle boundary. /// @@ -213,6 +192,31 @@ public struct Cycle: Sendable, Codable, Equatable, Hashable { self.init(year: UInt(year), month: UInt8(month), day: UInt8(day)) } + + /// Creates a cycle from date components that already fall on a cycle boundary. + /// + /// - Parameter components: Components carrying a year, month, and day. + private init?(dateComponents components: DateComponents) { + guard let year = components.year, + let month = components.month, + let day = components.day + else { + return nil + } + self.init(year: UInt(year), month: UInt8(month), day: UInt8(day)) + } + + /// The cycle whose start date is `days` away from this one's. + private func cycle(offsetByDays days: Int) -> Self? { + guard let firstDate, + let shiftedDate = Self.calendar.date(byAdding: .day, value: days, to: firstDate) + else { + return nil + } + return Self( + dateComponents: Self.calendar.dateComponents([.year, .month, .day], from: shiftedDate) + ) + } } extension Cycle: LosslessStringConvertible { @@ -237,7 +241,7 @@ extension Cycle: Comparable { extension Cycle: Identifiable { /// The unique identifier for this cycle in YYYYMMDD format. public var id: String { - String(format: "%04d%02d%02d", year, month, day) + unsafe String(format: "%04d%02d%02d", year, month, day) } } diff --git a/Sources/SwiftDOF/DOF.swift b/Sources/SwiftDOF/DOF.swift index 2bc231b..fcc0e06 100644 --- a/Sources/SwiftDOF/DOF.swift +++ b/Sources/SwiftDOF/DOF.swift @@ -37,7 +37,7 @@ public struct DOF: Sendable, Codable { data: Data, progressHandler: @Sendable (Progress) -> Void = { _ in }, errorCallback: ((any Error, Int) -> Void)? = nil - ) throws { + ) throws(DOFError) { var obstacles: [String: Obstacle] = [:] obstacles.reserveCapacity(Self.estimatedObstacleCount) @@ -84,7 +84,7 @@ public struct DOF: Sendable, Codable { url: URL, progressHandler: @Sendable (Progress) -> Void = { _ in }, errorCallback: ((any Error, Int) -> Void)? = nil - ) async throws { + ) async throws(DOFError) { var obstacles: [String: Obstacle] = [:] obstacles.reserveCapacity(Self.estimatedObstacleCount) @@ -180,6 +180,47 @@ public struct DOF: Sendable, Codable { self.obstaclesByID = obstacles } + /// Creates a DOF container by streaming a file from disk without holding it all in memory. + private init( + streamingFrom url: URL, + progressHandler: @Sendable (Progress) -> Void, + errorCallback: ((any Error, Int) -> Void)? + ) throws(DOFError) { + var obstacles: [String: Obstacle] = [:] + obstacles.reserveCapacity(Self.estimatedObstacleCount) + + var lineNumber = 0 + var cycle: Cycle? + + var reader = FileLineReader(url: url) + + // Setup progress tracking based on file size + let progress = Progress(totalUnitCount: reader.fileSize ?? -1) + progressHandler(progress) + + while let line = try reader.next() { + lineNumber += 1 + try Self.processLine( + line[...], + lineNumber: lineNumber, + cycle: &cycle, + obstacles: &obstacles, + errorCallback: errorCallback + ) + progress.completedUnitCount = reader.bytesRead + } + if progress.totalUnitCount > 0 { + progress.completedUnitCount = progress.totalUnitCount + } + + guard let cycle else { + throw DOFError.invalidFormat(.missingCurrencyDate) + } + + self.cycle = cycle + self.obstaclesByID = obstacles + } + /// Process a single line from the DOF file. private static func processLine( _ line: ArraySlice, @@ -187,7 +228,7 @@ public struct DOF: Sendable, Codable { cycle: inout Cycle?, obstacles: inout [String: Obstacle], errorCallback: ((any Error, Int) -> Void)? - ) throws { + ) throws(DOFError) { // Line 1: Parse currency date if lineNumber == 1 { cycle = try DOFByteParser.parseCurrencyDate(line) @@ -213,7 +254,7 @@ public struct DOF: Sendable, Codable { // MARK: - Static Factory Methods - /// Load DOF data from a file path (synchronous). + /// Load DOF data from a file path (synchronous), streaming the file from disk. /// /// - Parameters: /// - filePath: The URL of the DOF file. @@ -221,14 +262,17 @@ public struct DOF: Sendable, Codable { /// object that you can use to track parsing progress. /// - errorCallback: Optional callback for parse errors. /// - Returns: The parsed DOF. - /// - Throws: Error if loading or parsing fails. + /// - Throws: ``DOFError`` if loading or parsing fails. public static func from( filePath: URL, progressHandler: @Sendable (Progress) -> Void = { _ in }, errorCallback: ((any Error, Int) -> Void)? = nil - ) throws -> Self { - let data = try Data(contentsOf: filePath) - return try Self(data: data, progressHandler: progressHandler, errorCallback: errorCallback) + ) throws(DOFError) -> Self { + try Self( + streamingFrom: filePath, + progressHandler: progressHandler, + errorCallback: errorCallback + ) } /// Load DOF data from raw data. @@ -244,7 +288,7 @@ public struct DOF: Sendable, Codable { data: Data, progressHandler: @Sendable (Progress) -> Void = { _ in }, errorCallback: ((any Error, Int) -> Void)? = nil - ) throws -> Self { + ) throws(DOFError) -> Self { try Self(data: data, progressHandler: progressHandler, errorCallback: errorCallback) } @@ -261,7 +305,7 @@ public struct DOF: Sendable, Codable { url: URL, progressHandler: @Sendable (Progress) -> Void = { _ in }, errorCallback: ((any Error, Int) -> Void)? = nil - ) async throws -> Self { + ) async throws(DOFError) -> Self { try await Self(url: url, progressHandler: progressHandler, errorCallback: errorCallback) } diff --git a/Sources/SwiftDOF/Documentation.docc/Documentation.md b/Sources/SwiftDOF/Documentation.docc/Documentation.md index e329739..6803a48 100644 --- a/Sources/SwiftDOF/Documentation.docc/Documentation.md +++ b/Sources/SwiftDOF/Documentation.docc/Documentation.md @@ -43,7 +43,8 @@ the ``DOF/init(data:progressHandler:errorCallback:)`` callback parameter. - ``MarkingType`` - ``ActionCode`` - ``VerificationStatus`` -- ``AccuracyCategory`` +- ``HorizontalAccuracy`` +- ``VerticalAccuracy`` ### Errors diff --git a/Sources/SwiftDOF/Documentation.docc/Obstacle.md b/Sources/SwiftDOF/Documentation.docc/Obstacle.md index 5b44d89..7826577 100644 --- a/Sources/SwiftDOF/Documentation.docc/Obstacle.md +++ b/Sources/SwiftDOF/Documentation.docc/Obstacle.md @@ -32,7 +32,11 @@ - ``lighting`` - ``marking`` + +### Accuracy + - ``horizontalAccuracy`` +- ``verticalAccuracy`` ### Administrative diff --git a/Sources/SwiftDOF/Obstacle.swift b/Sources/SwiftDOF/Obstacle.swift index f657e9d..4b5758e 100644 --- a/Sources/SwiftDOF/Obstacle.swift +++ b/Sources/SwiftDOF/Obstacle.swift @@ -65,8 +65,11 @@ public struct Obstacle: Sendable, Codable { /// The type of lighting installed on the obstacle. public let lighting: LightingType - /// The horizontal accuracy category. - public let horizontalAccuracy: AccuracyCategory + /// The accuracy category of the obstacle's reported position. + public let horizontalAccuracy: HorizontalAccuracy + + /// The accuracy category of the obstacle's reported height. + public let verticalAccuracy: VerticalAccuracy /// The marking type (paint, flags, etc.). public let marking: MarkingType @@ -103,7 +106,8 @@ public struct Obstacle: Sendable, Codable { heightFtAGL: Int, heightFtMSL: Int, lighting: LightingType, - horizontalAccuracy: AccuracyCategory, + horizontalAccuracy: HorizontalAccuracy, + verticalAccuracy: VerticalAccuracy, marking: MarkingType, studyNumber: String, action: ActionCode, @@ -122,6 +126,7 @@ public struct Obstacle: Sendable, Codable { self.heightFtMSL = heightFtMSL self.lighting = lighting self.horizontalAccuracy = horizontalAccuracy + self.verticalAccuracy = verticalAccuracy self.marking = marking self.studyNumber = studyNumber self.action = action diff --git a/Sources/SwiftDOF/Parser/ASCII.swift b/Sources/SwiftDOF/Parser/ASCII.swift index 2d9011a..6cf4b23 100644 --- a/Sources/SwiftDOF/Parser/ASCII.swift +++ b/Sources/SwiftDOF/Parser/ASCII.swift @@ -11,8 +11,8 @@ enum ASCII { @usableFromInline static let slash: UInt8 = 0x2F // '/' @usableFromInline static let zero: UInt8 = 0x30 // '0' @usableFromInline static let nine: UInt8 = 0x39 // '9' - @usableFromInline static let A: UInt8 = 0x41 @usableFromInline static let E: UInt8 = 0x45 + @usableFromInline static let I: UInt8 = 0x49 @usableFromInline static let N: UInt8 = 0x4E @usableFromInline static let O: UInt8 = 0x4F @usableFromInline static let S: UInt8 = 0x53 diff --git a/Sources/SwiftDOF/Parser/ByteParsing.swift b/Sources/SwiftDOF/Parser/ByteParsing.swift index 5fd37ce..b652705 100644 --- a/Sources/SwiftDOF/Parser/ByteParsing.swift +++ b/Sources/SwiftDOF/Parser/ByteParsing.swift @@ -102,7 +102,7 @@ extension RandomAccessCollection where Element == UInt8, Index == Int { /// Convert to trimmed String (only when actually needed). /// - Throws: DOFError.invalidEncoding if bytes cannot be decoded as Latin-1. @inlinable - func toString() throws -> String { + func toString() throws(DOFError) -> String { guard let string = String(bytes: Array(self), encoding: .isoLatin1) else { throw DOFError.invalidEncoding } diff --git a/Sources/SwiftDOF/Parser/DOFByteParser.swift b/Sources/SwiftDOF/Parser/DOFByteParser.swift index 765eac1..a41daf0 100644 --- a/Sources/SwiftDOF/Parser/DOFByteParser.swift +++ b/Sources/SwiftDOF/Parser/DOFByteParser.swift @@ -23,9 +23,9 @@ struct DOFByteParser: Sendable { aglHeight: 83..<88, mslHeight: 89..<94, lighting: 95..<96, - accuracyH: 97..<98, - marking: 99..<100, - faaIndicator: 101..<102, + horizontalAccuracy: 97..<98, + verticalAccuracy: 99..<100, + marking: 101..<102, studyNumber: 103..<117, action: 118..<119, lastUpdated: 120..<127 @@ -34,8 +34,13 @@ struct DOFByteParser: Sendable { /// Minimum line length required for parsing. static let minimumLineLength = 127 - /// Pattern to match in currency date header. - private static let currencyDatePattern: [UInt8] = Array("CURRENCY DATE = ".utf8) + /// The bytes of "CURRENCY DATE = ", the pattern preceding the date in the DOF header. + private static let currencyDatePattern: InlineArray<16, UInt8> = [ + UInt8(ascii: "C"), UInt8(ascii: "U"), UInt8(ascii: "R"), UInt8(ascii: "R"), + UInt8(ascii: "E"), UInt8(ascii: "N"), UInt8(ascii: "C"), UInt8(ascii: "Y"), + UInt8(ascii: " "), UInt8(ascii: "D"), UInt8(ascii: "A"), UInt8(ascii: "T"), + UInt8(ascii: "E"), UInt8(ascii: " "), UInt8(ascii: "="), UInt8(ascii: " ") + ] // MARK: Public API @@ -43,7 +48,7 @@ struct DOFByteParser: Sendable { static func parseLine( _ bytes: T, lineNumber: Int = 0 - ) throws -> Obstacle where T.Element == UInt8, T.Index == Int { + ) throws(DOFError) -> Obstacle where T.Element == UInt8, T.Index == Int { guard bytes.count >= minimumLineLength else { throw DOFError.lineTooShort( expected: minimumLineLength, @@ -89,14 +94,29 @@ struct DOFByteParser: Sendable { ) } - // AccuracyCategory: space means unknown (category9) - let accuracyByte = bytes[base + fields.accuracyH.lowerBound] + // Both accuracy columns are blank when the FAA has not categorized the obstacle. + let horizontalAccuracyByte = bytes[base + fields.horizontalAccuracy.lowerBound] guard - let accuracy = AccuracyCategory(byte: accuracyByte == ASCII.space ? ASCII.nine : accuracyByte) + let horizontalAccuracy = HorizontalAccuracy( + byte: horizontalAccuracyByte == ASCII.space ? ASCII.nine : horizontalAccuracyByte + ) + else { + throw DOFError.parseError( + field: "horizontalAccuracy", + value: String(UnicodeScalar(horizontalAccuracyByte)), + line: lineNumber + ) + } + + let verticalAccuracyByte = bytes[base + fields.verticalAccuracy.lowerBound] + guard + let verticalAccuracy = VerticalAccuracy( + byte: verticalAccuracyByte == ASCII.space ? ASCII.I : verticalAccuracyByte + ) else { throw DOFError.parseError( - field: "accuracy", - value: String(UnicodeScalar(accuracyByte)), + field: "verticalAccuracy", + value: String(UnicodeScalar(verticalAccuracyByte)), line: lineNumber ) } @@ -110,11 +130,9 @@ struct DOFByteParser: Sendable { ) } - // MarkingType: 'N' and space are aliases for 'A' (none) + // MarkingType: a blank column means the marking is unknown. let markingByte = bytes[base + fields.marking.lowerBound] - let normalizedMarkingByte = - (markingByte == ASCII.N || markingByte == ASCII.space) ? ASCII.A : markingByte - guard let marking = MarkingType(byte: normalizedMarkingByte) else { + guard let marking = MarkingType(byte: markingByte == ASCII.space ? ASCII.U : markingByte) else { throw DOFError.parseError( field: "marking", value: String(UnicodeScalar(markingByte)), @@ -158,7 +176,8 @@ struct DOFByteParser: Sendable { heightFtAGL: agl, heightFtMSL: msl, lighting: lighting, - horizontalAccuracy: accuracy, + horizontalAccuracy: horizontalAccuracy, + verticalAccuracy: verticalAccuracy, marking: marking, studyNumber: studyNumber, action: action, @@ -170,26 +189,12 @@ struct DOFByteParser: Sendable { /// Expects format: "CURRENCY DATE = MM/DD/YY" static func parseCurrencyDate( _ bytes: T - ) throws -> Cycle where T.Element == UInt8, T.Index == Int { - // Find pattern - var matchStart: Int? - for i in bytes.startIndex..<(bytes.endIndex - currencyDatePattern.count) { - let slice = bytes[i..<(i + currencyDatePattern.count)] - guard zip(slice, currencyDatePattern).allSatisfy({ $0 == $1 }) else { continue } - matchStart = i + currencyDatePattern.count - break - } - - guard let start = matchStart else { + ) throws(DOFError) -> Cycle where T.Element == UInt8, T.Index == Int { + guard let start = currencyDateStart(in: bytes) else { throw DOFError.invalidFormat(.currencyDateHeaderNotFound) } - // Find slash positions in date portion - let dateBytes = bytes[start...] - var slashPositions: [Int] = [] - for (i, byte) in dateBytes.enumerated() where byte == ASCII.slash { - slashPositions.append(start + i) - } + let slashPositions = (start..= 2 else { throw DOFError.invalidFormat(.invalidCurrencyDateFormat) @@ -212,6 +217,25 @@ struct DOFByteParser: Sendable { // MARK: Private Helpers + /// The index just past the currency date pattern, or `nil` when the pattern is absent. + private static func currencyDateStart( + in bytes: T + ) -> Int? where T.Element == UInt8, T.Index == Int { + let patternLength = currencyDatePattern.count + guard bytes.count >= patternLength else { return nil } + + return (bytes.startIndex..<(bytes.endIndex - patternLength)) + .first { matchesCurrencyDatePattern(bytes, at: $0) } + .map { $0 + patternLength } + } + + private static func matchesCurrencyDatePattern( + _ bytes: T, + at start: Int + ) -> Bool where T.Element == UInt8, T.Index == Int { + currencyDatePattern.indices.allSatisfy { bytes[start + $0] == currencyDatePattern[$0] } + } + private static func slice( _ bytes: T, _ base: Int, @@ -224,7 +248,7 @@ struct DOFByteParser: Sendable { _ bytes: T, base: Int, lineNumber: Int - ) throws -> Double where T.Element == UInt8, T.Index == Int { + ) throws(DOFError) -> Double where T.Element == UInt8, T.Index == Int { let degSlice = slice(bytes, base, fields.latDegrees) let minSlice = slice(bytes, base, fields.latMinutes) let secSlice = slice(bytes, base, fields.latSeconds) @@ -268,7 +292,7 @@ struct DOFByteParser: Sendable { _ bytes: T, base: Int, lineNumber: Int - ) throws -> Double where T.Element == UInt8, T.Index == Int { + ) throws(DOFError) -> Double where T.Element == UInt8, T.Index == Int { let degSlice = slice(bytes, base, fields.lonDegrees) let minSlice = slice(bytes, base, fields.lonMinutes) let secSlice = slice(bytes, base, fields.lonSeconds) @@ -314,7 +338,7 @@ struct DOFByteParser: Sendable { private static func parseJulianDate( _ bytes: T, lineNumber: Int - ) throws -> DateComponents where T.Element == UInt8, T.Index == Int { + ) throws(DOFError) -> DateComponents where T.Element == UInt8, T.Index == Int { // Format: YYYYDDD (e.g., 2014138 = year 2014, day 138) let yearSlice = bytes.prefix(4) let daySlice = bytes.dropFirst(4) diff --git a/Sources/SwiftDOF/Parser/DOFLineReader.swift b/Sources/SwiftDOF/Parser/DOFLineReader.swift index 511902b..34af64e 100644 --- a/Sources/SwiftDOF/Parser/DOFLineReader.swift +++ b/Sources/SwiftDOF/Parser/DOFLineReader.swift @@ -26,10 +26,10 @@ struct DOFLineReader: Sequence, IteratorProtocol, Sendable { lineBuffer.removeAll(keepingCapacity: true) // Scan until LF or end of data - data.withUnsafeBytes { buffer in - let bytes = buffer.bindMemory(to: UInt8.self) + unsafe data.withUnsafeBytes { buffer in + let bytes = unsafe buffer.bindMemory(to: UInt8.self) while position < bytes.count { - let byte = bytes[position] + let byte = unsafe bytes[position] position += 1 if byte == ASCII.LF { return } lineBuffer.append(byte) @@ -45,13 +45,11 @@ struct DOFLineReader: Sequence, IteratorProtocol, Sendable { } } -// MARK: - AsyncDOFLineReader - -/// Async line reader for streaming DOF data from a file URL. -/// Reads in chunks to minimize memory usage for large files. -struct AsyncDOFLineReader: AsyncSequence, Sendable { - typealias Element = [UInt8] +// MARK: - FileLineReader +/// Line reader that streams DOF data from a file on disk. +/// Reads in chunks so a large file is never held in memory in its entirety. +struct FileLineReader: Sendable { /// Default read buffer size (64KB). static let defaultBufferSize = 65536 @@ -60,83 +58,117 @@ struct AsyncDOFLineReader: AsyncSequence, Sendable { private let url: URL private let bufferSize: Int + private var handle: FileHandle? + private var buffer: [UInt8] = [] + private var bufferPosition = 0 + private var lineBuffer: [UInt8] = [] + private var isAtEnd = false /// The total size of the file in bytes, if known. let fileSize: Int64? + /// Total bytes read from the file so far. + private(set) var bytesRead: Int64 = 0 + + private var bufferIsExhausted: Bool { bufferPosition >= buffer.count } + init(url: URL, bufferSize: Int = defaultBufferSize) { self.url = url self.bufferSize = bufferSize - // Try to get file size for progress tracking - if let attrs = try? FileManager.default.attributesOfItem(atPath: url.path), - let size = attrs[.size] as? Int64 - { - self.fileSize = size - } else { - self.fileSize = nil - } + self.fileSize = Self.sizeOfFile(at: url) + lineBuffer.reserveCapacity(Self.lineBufferCapacity) } - func makeAsyncIterator() -> AsyncIterator { - AsyncIterator(url: url, bufferSize: bufferSize) + /// The size in bytes of the file at `url`, if it can be determined. + static func sizeOfFile(at url: URL) -> Int64? { + guard let attributes = try? FileManager.default.attributesOfItem(atPath: url.path) else { + return nil + } + return attributes[.size] as? Int64 } - struct AsyncIterator: AsyncIteratorProtocol { - private let url: URL - private let bufferSize: Int - private var handle: FileHandle? - private var buffer: [UInt8] = [] - private var bufferPos: Int = 0 - private var lineBuffer: [UInt8] = [] - private var isEOF = false + /// Returns the next line, or `nil` once the file is exhausted. + mutating func next() throws(DOFError) -> [UInt8]? { + guard !isAtEnd else { return nil } - /// Total bytes read from the file so far. - private(set) var bytesRead: Int64 = 0 + let handle = try openedHandle() + lineBuffer.removeAll(keepingCapacity: true) - init(url: URL, bufferSize: Int) { - self.url = url - self.bufferSize = bufferSize - self.lineBuffer.reserveCapacity(lineBufferCapacity) - } + while true { + if bufferIsExhausted { + guard let chunk = try readChunk(from: handle), !chunk.isEmpty else { + isAtEnd = true + return lineBuffer.isEmpty ? nil : lineBuffer + } + bytesRead += Int64(chunk.count) + buffer = Array(chunk) + bufferPosition = 0 + } - mutating func next() throws -> [UInt8]? { - guard !isEOF else { return nil } + let byte = buffer[bufferPosition] + bufferPosition += 1 - // Lazily open file handle on first call - if handle == nil { - handle = try FileHandle(forReadingFrom: url) + if byte == ASCII.LF { + // Strip trailing CR if present (handles CRLF) + if lineBuffer.last == ASCII.CR { + lineBuffer.removeLast() + } + return lineBuffer } - guard let handle else { preconditionFailure("handle was nil") } + lineBuffer.append(byte) + } + } - lineBuffer.removeAll(keepingCapacity: true) + private mutating func openedHandle() throws(DOFError) -> FileHandle { + if let handle { return handle } + guard let opened = try? FileHandle(forReadingFrom: url) else { + throw DOFError.fileNotFound(url) + } + handle = opened + return opened + } - while true { - // Refill buffer if exhausted - if bufferPos >= buffer.count { - guard let chunk = try handle.read(upToCount: bufferSize), - !chunk.isEmpty - else { - isEOF = true - // Return any remaining content as final line - return lineBuffer.isEmpty ? nil : lineBuffer - } - bytesRead += Int64(chunk.count) - buffer = Array(chunk) - bufferPos = 0 - } + private func readChunk(from handle: FileHandle) throws(DOFError) -> Data? { + do { + return try handle.read(upToCount: bufferSize) + } catch { + throw DOFError.streamError(error) + } + } +} + +// MARK: - AsyncDOFLineReader - let byte = buffer[bufferPos] - bufferPos += 1 +/// Async façade over ``FileLineReader`` for `for await` iteration of a DOF file. +struct AsyncDOFLineReader: AsyncSequence, Sendable { + typealias Element = [UInt8] + typealias Failure = DOFError - if byte == ASCII.LF { - // Strip trailing CR if present (handles CRLF) - if lineBuffer.last == ASCII.CR { - lineBuffer.removeLast() - } - return lineBuffer - } - lineBuffer.append(byte) - } + private let url: URL + private let bufferSize: Int + + /// The total size of the file in bytes, if known. + let fileSize: Int64? + + init(url: URL, bufferSize: Int = FileLineReader.defaultBufferSize) { + self.url = url + self.bufferSize = bufferSize + self.fileSize = FileLineReader.sizeOfFile(at: url) + } + + func makeAsyncIterator() -> AsyncIterator { + AsyncIterator(reader: FileLineReader(url: url, bufferSize: bufferSize)) + } + + struct AsyncIterator: AsyncIteratorProtocol { + private var reader: FileLineReader + + init(reader: FileLineReader) { + self.reader = reader + } + + mutating func next() throws(DOFError) -> [UInt8]? { + try reader.next() } } } @@ -148,6 +180,7 @@ struct AsyncDOFLineReader: AsyncSequence, Sendable { struct AsyncBytesLineReader: AsyncSequence, Sendable where Source.Element == UInt8, Source: Sendable { typealias Element = [UInt8] + typealias Failure = Source.Failure /// Pre-allocated capacity for line buffer (DOF lines are ~128 bytes). private static var lineBufferCapacity: Int { 256 } @@ -172,10 +205,10 @@ where Source.Element == UInt8, Source: Sendable { } @concurrent - mutating func next() async throws -> [UInt8]? { + mutating func next() async throws(Source.Failure) -> [UInt8]? { lineBuffer.removeAll(keepingCapacity: true) - while let byte = try await iterator.next() { + while let byte = try await iterator.next(isolation: nil) { if byte == ASCII.LF { // Strip trailing CR if present (handles CRLF) if lineBuffer.last == ASCII.CR { diff --git a/Sources/SwiftDOF/Types/AccuracyCategory.swift b/Sources/SwiftDOF/Types/AccuracyCategory.swift deleted file mode 100644 index 164ca93..0000000 --- a/Sources/SwiftDOF/Types/AccuracyCategory.swift +++ /dev/null @@ -1,53 +0,0 @@ -public import Foundation - -/// FAA horizontal accuracy category for obstacle position data. -/// -/// Categories 1-9 and A indicate the accuracy of the obstacle's reported position. -/// Lower numbers indicate higher accuracy. -public enum AccuracyCategory: Character, Sendable, Codable, CaseIterable, ByteInitializable { - /// Survey data accuracy (approximately ±3 feet). - case survey = "A" - - /// ±20 feet horizontal accuracy. - case category1 = "1" - - /// ±50 feet horizontal accuracy. - case category2 = "2" - - /// ±100 feet horizontal accuracy. - case category3 = "3" - - /// ±250 feet horizontal accuracy. - case category4 = "4" - - /// ±500 feet horizontal accuracy. - case category5 = "5" - - /// ±1,000 feet horizontal accuracy. - case category6 = "6" - - /// ±0.5 nautical mile horizontal accuracy. - case category7 = "7" - - /// ±1 nautical mile horizontal accuracy. - case category8 = "8" - - /// Unknown accuracy. - case category9 = "9" - - /// Approximate accuracy, if known. - public var accuracy: Measurement? { - switch self { - case .survey: .init(value: 3, unit: .feet) - case .category1: .init(value: 20, unit: .feet) - case .category2: .init(value: 50, unit: .feet) - case .category3: .init(value: 100, unit: .feet) - case .category4: .init(value: 250, unit: .feet) - case .category5: .init(value: 500, unit: .feet) - case .category6: .init(value: 1000, unit: .feet) - case .category7: .init(value: 0.5, unit: .nauticalMiles) - case .category8: .init(value: 1, unit: .nauticalMiles) - case .category9: nil - } - } -} diff --git a/Sources/SwiftDOF/Types/HorizontalAccuracy.swift b/Sources/SwiftDOF/Types/HorizontalAccuracy.swift new file mode 100644 index 0000000..a8ba5e6 --- /dev/null +++ b/Sources/SwiftDOF/Types/HorizontalAccuracy.swift @@ -0,0 +1,51 @@ +public import Foundation + +/// FAA horizontal accuracy code for an obstacle's reported position. +/// +/// The FAA has used these codes since 1979 to declare how tightly an obstacle's +/// location is known. Lower code numbers indicate a more accurate position. A code +/// is typically paired with its ``VerticalAccuracy`` counterpart and spoken as a +/// single accuracy code, such as "1A" or "4D". +public enum HorizontalAccuracy: Character, Sendable, Codable, CaseIterable, ByteInitializable { + /// ±20 feet horizontal accuracy. + case code1 = "1" + + /// ±50 feet horizontal accuracy. + case code2 = "2" + + /// ±100 feet horizontal accuracy. + case code3 = "3" + + /// ±250 feet horizontal accuracy. + case code4 = "4" + + /// ±500 feet horizontal accuracy. + case code5 = "5" + + /// ±1,000 feet horizontal accuracy. + case code6 = "6" + + /// ±0.5 nautical mile horizontal accuracy. + case code7 = "7" + + /// ±1 nautical mile horizontal accuracy. + case code8 = "8" + + /// Unknown accuracy. + case code9 = "9" + + /// The tolerance this code guarantees, or `nil` when the accuracy is unknown. + public var tolerance: Measurement? { + switch self { + case .code1: .init(value: 20, unit: .feet) + case .code2: .init(value: 50, unit: .feet) + case .code3: .init(value: 100, unit: .feet) + case .code4: .init(value: 250, unit: .feet) + case .code5: .init(value: 500, unit: .feet) + case .code6: .init(value: 1000, unit: .feet) + case .code7: .init(value: 0.5, unit: .nauticalMiles) + case .code8: .init(value: 1, unit: .nauticalMiles) + case .code9: nil + } + } +} diff --git a/Sources/SwiftDOF/Types/MarkingType.swift b/Sources/SwiftDOF/Types/MarkingType.swift index c721901..c1748f4 100644 --- a/Sources/SwiftDOF/Types/MarkingType.swift +++ b/Sources/SwiftDOF/Types/MarkingType.swift @@ -1,33 +1,27 @@ import Foundation -/// Obstacle marking indicator from the DOF. +/// Obstacle mark indicator from the DOF. /// /// Indicates how an obstacle is marked for visibility to aircraft. public enum MarkingType: Character, Sendable, Codable, CaseIterable, ByteInitializable { - /// No marking or unknown. - case none = "A" + /// Marked with orange, or orange and white, paint. + case orangeOrOrangeWhitePaint = "P" - /// Marked with high visibility orange and white paint. - case orangeWhitePaint = "B" + /// Marked with white paint only. + case whitePaintOnly = "W" - /// Marked with flag markers. - case flagMarkers = "C" - - /// Marked with orange and white paint and has flag markers. - case paintAndFlags = "D" - - /// Marked with high visibility lighting only. - case lightingOnly = "E" + /// Marked, by a means the DOF does not identify. + case marked = "M" - /// Marked with orange and white paint and high visibility lighting. - case paintAndLighting = "F" + /// Marked with flag markers. + case flagMarker = "F" - /// Marked with flag markers and high visibility lighting. - case flagsAndLighting = "G" + /// Marked with spherical markers (typically on power lines). + case sphericalMarker = "S" - /// Marked with orange and white paint, flag markers, and high visibility lighting. - case paintFlagsAndLighting = "H" + /// Not marked. + case none = "N" - /// Marked with spherical markers (typically on power lines). - case sphericalMarkers = "I" + /// Marking unknown. + case unknown = "U" } diff --git a/Sources/SwiftDOF/Types/VerticalAccuracy.swift b/Sources/SwiftDOF/Types/VerticalAccuracy.swift new file mode 100644 index 0000000..1282a9b --- /dev/null +++ b/Sources/SwiftDOF/Types/VerticalAccuracy.swift @@ -0,0 +1,51 @@ +public import Foundation + +/// FAA vertical accuracy code for an obstacle's reported height. +/// +/// The FAA has used these codes since 1979 to declare how tightly an obstacle's +/// height is known. Earlier code letters indicate a more accurate height. A code +/// is typically paired with its ``HorizontalAccuracy`` counterpart and spoken as a +/// single accuracy code, such as "1A" or "4D". +public enum VerticalAccuracy: Character, Sendable, Codable, CaseIterable, ByteInitializable { + /// ±3 feet vertical accuracy. + case codeA = "A" + + /// ±10 feet vertical accuracy. + case codeB = "B" + + /// ±20 feet vertical accuracy. + case codeC = "C" + + /// ±50 feet vertical accuracy. + case codeD = "D" + + /// ±125 feet vertical accuracy. + case codeE = "E" + + /// ±250 feet vertical accuracy. + case codeF = "F" + + /// ±500 feet vertical accuracy. + case codeG = "G" + + /// ±1,000 feet vertical accuracy. + case codeH = "H" + + /// Unknown accuracy. + case codeI = "I" + + /// The tolerance this code guarantees, or `nil` when the accuracy is unknown. + public var tolerance: Measurement? { + switch self { + case .codeA: .init(value: 3, unit: .feet) + case .codeB: .init(value: 10, unit: .feet) + case .codeC: .init(value: 20, unit: .feet) + case .codeD: .init(value: 50, unit: .feet) + case .codeE: .init(value: 125, unit: .feet) + case .codeF: .init(value: 250, unit: .feet) + case .codeG: .init(value: 500, unit: .feet) + case .codeH: .init(value: 1000, unit: .feet) + case .codeI: nil + } + } +} diff --git a/Sources/SwiftDOF_E2E/OutputFormatter.swift b/Sources/SwiftDOF_E2E/OutputFormatter.swift index 0c6f5c4..1f79e19 100644 --- a/Sources/SwiftDOF_E2E/OutputFormatter.swift +++ b/Sources/SwiftDOF_E2E/OutputFormatter.swift @@ -17,9 +17,10 @@ protocol OutputFormatter { extension OutputStream { func write(_ string: String) { guard let data = string.data(using: .utf8) else { return } - data.withUnsafeBytes { buffer in - guard let pointer = buffer.baseAddress?.assumingMemoryBound(to: UInt8.self) else { return } - write(pointer, maxLength: buffer.count) + unsafe data.withUnsafeBytes { buffer in + guard let pointer = unsafe buffer.baseAddress?.assumingMemoryBound(to: UInt8.self) + else { return } + unsafe write(pointer, maxLength: buffer.count) } } @@ -48,9 +49,10 @@ struct JSONOutputFormatter: OutputFormatter { let encoder = JSONEncoder() encoder.outputFormatting = [.prettyPrinted, .sortedKeys] let jsonData = try encoder.encode(dof.all) - jsonData.withUnsafeBytes { buffer in - guard let pointer = buffer.baseAddress?.assumingMemoryBound(to: UInt8.self) else { return } - stream.write(pointer, maxLength: buffer.count) + unsafe jsonData.withUnsafeBytes { buffer in + guard let pointer = unsafe buffer.baseAddress?.assumingMemoryBound(to: UInt8.self) + else { return } + unsafe stream.write(pointer, maxLength: buffer.count) } } } diff --git a/Sources/SwiftDOF_E2E/Report.swift b/Sources/SwiftDOF_E2E/Report.swift new file mode 100644 index 0000000..a9ab5d1 --- /dev/null +++ b/Sources/SwiftDOF_E2E/Report.swift @@ -0,0 +1,200 @@ +import Foundation +import SwiftDOF + +/// A machine-readable record of one end-to-end parse, written with `--report`. +/// +/// The report is what the cycle-watch workflow reads to decide whether a cycle parsed cleanly, +/// and it doubles as the baseline the next cycle is compared against. +struct Report: Codable { + + /// The share of the total obstacle count that may change between cycles before it counts as + /// drift. Obstacle data turns over slowly; a swing this large is a parsing problem, not news. + private static let totalDriftTolerance = 0.05 + + /// The share of a single region's obstacle count that may change between cycles. Regions are + /// small enough to be noisier than the file as a whole. + private static let regionDriftTolerance = 0.10 + + /// The greatest number of parse errors quoted in the report. + private static let maximumErrorSamples = 50 + + /// The cycle the parsed file declares in its currency date header. + let cycle: String + + /// Where the data was read from. + let source: String + + /// The number of obstacles parsed. + let obstacleCount: Int + + /// The number of lines that failed to parse. + let parseErrorCount: Int + + /// How long loading and parsing took, in seconds. + let elapsed: TimeInterval + + /// Obstacle counts keyed by region, as `COUNTRY-STATE` or bare `COUNTRY` where the DOF carries + /// no state. The country is always part of the key because state and country codes collide — + /// `CA` is both California and Canada. + let countsByRegion: [String: Int] + + /// Up to ``maximumErrorSamples`` of the parse errors. + let errorSamples: [ErrorSample] + + /// Whether this cycle failed its check. + let failed: Bool + + /// Why the cycle failed, empty when it did not. + let failureReasons: [String] + + /// Counts that moved more than their tolerance against the baseline. + let drift: [Drift] + + /// Builds a report from a completed parse, optionally compared against an earlier cycle's report. + init( + dof: DOF, + source: String, + parseErrorCount: Int, + errorSamples: [ErrorSample], + elapsed: TimeInterval, + baseline: Self? + ) { + let countsByRegion = Self.countsByRegion(of: dof) + + self.cycle = dof.cycle.id + self.source = source + self.obstacleCount = dof.count + self.parseErrorCount = parseErrorCount + self.elapsed = elapsed + self.countsByRegion = countsByRegion + self.errorSamples = Array(errorSamples.prefix(Self.maximumErrorSamples)) + self.failureReasons = Self.failureReasons( + obstacleCount: dof.count, + parseErrorCount: parseErrorCount + ) + self.failed = !failureReasons.isEmpty + self.drift = + baseline.map { + Self.drift(from: $0, totalNow: dof.count, regionsNow: countsByRegion) + } ?? [] + } + + /// Reads a report written by an earlier run. + static func read(from url: URL) throws -> Self { + try JSONDecoder().decode(Self.self, from: Data(contentsOf: url)) + } + + /// Obstacle counts keyed by country and state, so that California and Canada stay apart. + private static func countsByRegion(of dof: DOF) -> [String: Int] { + dof.reduce(into: [:]) { counts, obstacle in + counts[regionKey(of: obstacle), default: 0] += 1 + } + } + + private static func regionKey(of obstacle: Obstacle) -> String { + guard let state = obstacle.state else { return obstacle.country } + return "\(obstacle.country)-\(state)" + } + + private static func failureReasons(obstacleCount: Int, parseErrorCount: Int) -> [String] { + var reasons: [String] = [] + if parseErrorCount > 0 { + reasons.append("\(parseErrorCount) line(s) failed to parse.") + } + if obstacleCount == 0 { + reasons.append("The file parsed to zero obstacles.") + } + return reasons + } + + private static func drift( + from baseline: Self, + totalNow: Int, + regionsNow: [String: Int] + ) -> [Drift] { + var drift: [Drift] = [] + + if let total = Drift( + scope: "total", + was: baseline.obstacleCount, + now: totalNow, + tolerance: totalDriftTolerance + ) { + drift.append(total) + } + + for (region, was) in baseline.countsByRegion.sorted(by: { $0.key < $1.key }) { + if let regionDrift = Drift( + scope: region, + was: was, + now: regionsNow[region], + tolerance: regionDriftTolerance + ) { + drift.append(regionDrift) + } + } + + return drift + } + + /// One parse error, quoted for the report. + struct ErrorSample: Codable { + /// The line the error was reported on. + let line: Int + + /// The error's general description. + let message: String + + /// Which field failed and why, when the error carries that detail. + let reason: String? + + init(line: Int, error: any Error) { + self.line = line + self.message = error.localizedDescription + self.reason = (error as? (any LocalizedError))?.failureReason + } + } + + /// A count that moved more than its tolerance between two cycles. + struct Drift: Codable { + /// What the count covers: `total`, or a region key from ``Report/countsByRegion``. + let scope: String + + /// The count in the baseline cycle. + let was: Int + + /// The count in this cycle, or `nil` when the scope vanished entirely. + let now: Int? + + /// Why the movement was recorded. + let reason: String + + /// Records drift, or returns `nil` when the count held within `tolerance`. + init?(scope: String, was: Int, now: Int?, tolerance: Double) { + guard let now else { + self.init(scope: scope, was: was, now: nil, reason: "The region is no longer present.") + return + } + + guard was > 0 else { return nil } + + let change = Double(now - was) / Double(was) + guard abs(change) > tolerance else { return nil } + + let percent = (abs(change) * 100).formatted(.number.precision(.fractionLength(1))) + self.init( + scope: scope, + was: was, + now: now, + reason: "The count \(change < 0 ? "fell" : "rose") by \(percent)%." + ) + } + + private init(scope: String, was: Int, now: Int?, reason: String) { + self.scope = scope + self.was = was + self.now = now + self.reason = reason + } + } +} diff --git a/Sources/SwiftDOF_E2E/SwiftDOF_E2E.swift b/Sources/SwiftDOF_E2E/SwiftDOF_E2E.swift index 9d87e91..4f324b6 100644 --- a/Sources/SwiftDOF_E2E/SwiftDOF_E2E.swift +++ b/Sources/SwiftDOF_E2E/SwiftDOF_E2E.swift @@ -32,6 +32,22 @@ struct SwiftDOF_E2E: AsyncParsableCommand { @Option(name: .shortAndLong, help: "Output format: summary or json") var format: OutputFormat = .summary + @Option( + name: .long, + help: "Path to write a JSON report of the parse to", + completion: .file(extensions: ["json"]), + transform: { URL(filePath: $0) } + ) + var report: URL? + + @Option( + name: .long, + help: "Path to an earlier report to compare counts against", + completion: .file(extensions: ["json"]), + transform: { URL(filePath: $0) } + ) + var baseline: URL? + private var currentCycleURL: URL { get throws { let cycle = Cycle.effective @@ -42,7 +58,7 @@ struct SwiftDOF_E2E: AsyncParsableCommand { else { fatalError("Current cycle could not be determined") } - let filename = String(format: "DOF_%02d%02d%02d.zip", year % 100, month, day) + let filename = unsafe String(format: "DOF_%02d%02d%02d.zip", year % 100, month, day) guard let url = URL(string: "https://aeronav.faa.gov/Obst_Data/\(filename)") else { fatalError("Current DOF URL could not be determined") } @@ -56,6 +72,7 @@ struct SwiftDOF_E2E: AsyncParsableCommand { let formatter = makeFormatter(for: format) var errorCount = 0 + var errorSamples: [Report.ErrorSample] = [] let startTime = Date() let showProgress = format != .json @@ -70,6 +87,7 @@ struct SwiftDOF_E2E: AsyncParsableCommand { }, errorCallback: { error, line in errorCount += 1 + errorSamples.append(.init(line: line, error: error)) var message = "Error at line \(line): \(error.localizedDescription)" if let reason = (error as? (any LocalizedError))?.failureReason { message += "\n - \(reason)" @@ -93,6 +111,44 @@ struct SwiftDOF_E2E: AsyncParsableCommand { defer { stdout.close() } try formatter.format(dof: dof, errorCount: errorCount, elapsed: elapsed, to: stdout) + + if let report { + try writeReport( + to: report, + dof: dof, + source: inputURL, + errorCount: errorCount, + errorSamples: errorSamples, + elapsed: elapsed + ) + } + + if errorCount > 0 { + throw ExitCode.failure + } + } + + /// Writes the JSON report, comparing counts against `baseline` when one was given. + private func writeReport( + to url: URL, + dof: DOF, + source: URL, + errorCount: Int, + errorSamples: [Report.ErrorSample], + elapsed: TimeInterval + ) throws { + let report = Report( + dof: dof, + source: source.absoluteString, + parseErrorCount: errorCount, + errorSamples: errorSamples, + elapsed: elapsed, + baseline: try baseline.map { try Report.read(from: $0) } + ) + + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + try encoder.encode(report).write(to: url) } private func makeLoader(for url: URL) -> any DOFDataLoader { diff --git a/Tests/SwiftDOFTests/DOFTests.swift b/Tests/SwiftDOFTests/DOFTests.swift index 1c1ad44..e49ffb1 100644 --- a/Tests/SwiftDOFTests/DOFTests.swift +++ b/Tests/SwiftDOFTests/DOFTests.swift @@ -23,6 +23,11 @@ struct DOFTests { sampleDOFContent.data(using: .utf8)! } + /// A URL in the temporary directory that no file occupies. + private static func temporaryFileURL() -> URL { + FileManager.default.temporaryDirectory.appendingPathComponent("\(UUID().uuidString).dat") + } + @Test func `parses obstacles and the currency date from DOF data`() throws { let dof = try DOF(data: sampleDOFData) @@ -172,6 +177,41 @@ struct DOFTests { #expect(dof.count == 3) } + @Test + func `parses obstacles streamed from a file on disk`() throws { + let dof = try withTemporaryFile(containing: sampleDOFContent) { + try DOF.from(filePath: $0) + } + + #expect(dof.count == 3) + #expect(dof.cycle.year == 2025) + } + + @Test(arguments: [1, 7, 64, 4096]) + func `reads the same lines from a file whatever the buffer size`(_ bufferSize: Int) throws { + let lines = try withTemporaryFile(containing: sampleDOFContent) { url in + var reader = FileLineReader(url: url, bufferSize: bufferSize) + var lines: [[UInt8]] = [] + while let line = try reader.next() { + lines.append(line) + } + return lines + } + + #expect(lines == sampleDOFContent.split(separator: "\n").map { Array($0.utf8) }) + } + + @Test + func `throws a file-not-found error for a missing file`() { + let missingFile = Self.temporaryFileURL() + + let error = #expect(throws: DOFError.self) { + try DOF.from(filePath: missingFile) + } + + #expect(error?.isFileNotFound == true) + } + @Test func `leaves the error callback uncalled for valid data`() throws { var errorCalled = false @@ -186,4 +226,22 @@ struct DOFTests { #expect(dof.count == 3) #expect(!errorCalled) // No errors in valid content } + + /// Writes `content` to a temporary file, hands its URL to `body`, and removes the file after. + private func withTemporaryFile( + containing content: String, + _ body: (URL) throws -> T + ) throws -> T { + let url = Self.temporaryFileURL() + try content.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + return try body(url) + } +} + +extension DOFError { + fileprivate var isFileNotFound: Bool { + if case .fileNotFound = self { return true } + return false + } } diff --git a/Tests/SwiftDOFTests/ObstacleTests.swift b/Tests/SwiftDOFTests/ObstacleTests.swift index 74ec3f1..346be39 100644 --- a/Tests/SwiftDOFTests/ObstacleTests.swift +++ b/Tests/SwiftDOFTests/ObstacleTests.swift @@ -9,11 +9,17 @@ import Foundation struct ObstacleTests { - // Sample DOF record line as bytes - let sampleLineBytes: [UInt8] = Array( + // Sample DOF record line + let sampleLine = "01-001307 O US AL DAUPHIN ISLAND 30 10 45.00N 088 04 39.00W RIG 1 00236 00236 R 5 D M 1990ASO01578OE C 2014138 " - .utf8 - ) + + var sampleLineBytes: [UInt8] { Array(sampleLine.utf8) } + + /// The range of a single one-indexed DOF column, as the FAA README numbers them. + private func column(_ oneIndexed: Int) -> Range { + let start = sampleLine.index(sampleLine.startIndex, offsetBy: oneIndexed - 1) + return start..