diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 7a1751c..c4f787e 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -1,18 +1,30 @@ -name: Build NPK and container release +name: Build RouterOS package release on: push: - tags: ['ros-*-arm-r*'] + tags: ['ros-*-arm-r*', 'ros-*-arm64-r*'] workflow_dispatch: inputs: version: - description: RouterOS version (ARM) + description: RouterOS version required: true default: '7.25.1' + arch: + description: RouterOS package architecture + required: true + type: choice + options: [arm, arm64] + default: arm + wifi: + description: Optional Wi-Fi package (choose one compatible with the architecture) + required: true + type: choice + options: [none, wifi-qcom-ac, wifi-qcom, wireless] + default: none tag: - description: Unique prerelease tag, e.g. ros-7.25.1-arm-r1 + description: Unique prerelease tag, e.g. ros-7.25.1-arm64-r1 required: true publish: - description: Publish containers to registry and create a Gitea prerelease + description: Create a Gitea prerelease after building the package set type: boolean default: false jobs: @@ -35,18 +47,30 @@ jobs: REF_TYPE: ${{ github.ref_type }} INPUT_VERSION: ${{ inputs.version }} INPUT_TAG: ${{ inputs.tag }} + INPUT_ARCH: ${{ inputs.arch }} + INPUT_WIFI: ${{ inputs.wifi }} run: | python3 - <<'PYTHON' import os, re tag = os.environ['REF_NAME'] if os.environ['REF_TYPE'] == 'tag' else os.environ['INPUT_TAG'] - match = re.fullmatch(r'ros-([0-9]+\.[0-9]+(?:\.[0-9]+)?)-arm-r[0-9]+', tag) + match = re.fullmatch(r'ros-([0-9]+\.[0-9]+(?:\.[0-9]+)?)-(arm64|arm)-r[0-9]+', tag) if not match: - raise SystemExit('Expected release tag ros-VERSION-arm-rNUMBER') - version = match[1] + raise SystemExit('Expected release tag ros-VERSION-arm-rNUMBER or ros-VERSION-arm64-rNUMBER') + version, arch = match[1], match[2] if os.environ['INPUT_VERSION'] and os.environ['INPUT_VERSION'] != version: raise SystemExit('Tag version and requested RouterOS version differ') + if os.environ['REF_TYPE'] != 'tag' and os.environ['INPUT_ARCH'] != arch: + raise SystemExit('Tag architecture and selected architecture differ') + if os.environ['REF_TYPE'] != 'tag': + wifi = os.environ.get('INPUT_WIFI', 'none') or 'none' + else: + wifi = 'wifi-qcom-ac' if arch == 'arm' else 'wifi-qcom' + if wifi == 'wifi-qcom-ac' and arch != 'arm': + raise SystemExit('wifi-qcom-ac requires arm') + if wifi == 'wifi-qcom' and arch != 'arm64': + raise SystemExit('wifi-qcom requires arm64') with open(os.environ['GITHUB_OUTPUT'], 'a') as output: - output.write(f'version={version}\ntag={tag}\n') + output.write(f'version={version}\narch={arch}\nwifi={wifi}\ntag={tag}\n') PYTHON - name: Check signing secret env: @@ -55,15 +79,12 @@ jobs: - name: Check publication secrets if: startsWith(github.ref, 'refs/tags/ros-') || inputs.publish == true || inputs.publish == 'true' env: - QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} - QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} GITEA_RELEASE_TOKEN: ${{ secrets.GITEA_RELEASE_TOKEN || secrets.GITEA_TOKEN }} run: | - test -n "$QUAY_USERNAME" && test -n "$QUAY_TOKEN" && test -n "$GITEA_RELEASE_TOKEN" || { - echo 'Configure registry credentials and allow the job token repository release write access.' + test -n "$GITEA_RELEASE_TOKEN" || { + echo 'Configure GITEA_RELEASE_TOKEN or allow the job token repository release write access.' exit 1 } - tools/build-container.sh check-registry - name: Use prepared build environment run: | cmp requirements.txt /opt/mikrotik-olsr-requirements.txt || { echo 'Rebuild ci/Dockerfile for the changed dependencies.'; exit 1; } @@ -77,7 +98,10 @@ jobs: build/runtime-sources/*.tar.bz2 build/runtime-sources/*.tar.gz build/olsrd-source/.git - key: olsr-sources-v1-${{ hashFiles('tools/build-runtime.sh', 'tools/build-olsrd.sh', 'tools/build-bird.sh') }} + key: olsr-sources-v2-${{ steps.parameters.outputs.arch }}-${{ hashFiles('tools/build-runtime.sh', 'tools/build-olsrd.sh', 'tools/build-bird.sh') }} + restore-keys: | + olsr-sources-v2-${{ steps.parameters.outputs.arch }}- + olsr-sources-v2- - name: Restore official RouterOS inputs id: inputs-cache uses: https://github.com/actions/cache/restore@v3 @@ -86,7 +110,7 @@ jobs: build/inputs-*/*.npk build/inputs-*/*.tar.gz build/inputs-*/*.zip - key: routeros-inputs-v1-${{ steps.parameters.outputs.version }}-arm + key: routeros-inputs-v2-${{ steps.parameters.outputs.version }}-${{ steps.parameters.outputs.arch }} - name: Test source run: | set -eu @@ -95,7 +119,8 @@ jobs: - name: Build open-source payloads env: ROUTEROS_VERSION: ${{ steps.parameters.outputs.version }} - run: tools/build-runtime.sh + ROUTEROS_ARCH: ${{ steps.parameters.outputs.arch }} + run: tools/build-runtime.sh "$ROUTEROS_ARCH" - name: Save verified source downloads if: steps.sources-cache.outputs.cache-hit != 'true' uses: https://github.com/actions/cache/save@v3 @@ -110,6 +135,8 @@ jobs: env: SIGNING_KEYSET_JSON: ${{ secrets.MIKROTIK_SIGNING_KEYSET }} ROUTEROS_VERSION: ${{ steps.parameters.outputs.version }} + ROUTEROS_ARCH: ${{ steps.parameters.outputs.arch }} + WIFI_PACKAGE: ${{ steps.parameters.outputs.wifi }} run: | set -eu keyfile=$(mktemp) @@ -117,9 +144,12 @@ jobs: chmod 600 "$keyfile" printf '%s' "$SIGNING_KEYSET_JSON" > "$keyfile" unset SIGNING_KEYSET_JSON - .venv/bin/python tools/build-release.py --version "$ROUTEROS_VERSION" \ - --arch arm --wifi wifi-qcom-ac --keys "$keyfile" --output dist \ - --option-root build/option-root --olsr-root build/olsr-root + set -- .venv/bin/python tools/build-release.py --version "$ROUTEROS_VERSION" \ + --arch "$ROUTEROS_ARCH" --keys "$keyfile" --output dist \ + --option-root "build/runtime-$ROUTEROS_ARCH/option-root" \ + --olsr-root "build/runtime-$ROUTEROS_ARCH/olsr-root" + if [ "$WIFI_PACKAGE" != none ]; then set -- "$@" --wifi "$WIFI_PACKAGE"; fi + "$@" cd dist sha256sum -c SHA256SUMS - name: Save official RouterOS inputs @@ -131,23 +161,6 @@ jobs: build/inputs-*/*.tar.gz build/inputs-*/*.zip key: ${{ steps.inputs-cache.outputs.cache-primary-key }} - - name: Build ARMv7 and ARM64 containers - run: tools/build-container.sh build - - name: Combine release assets - run: | - set -eu - cp build/container-artifacts/* dist/ - .venv/bin/python - <<'PYTHON' - import hashlib - from pathlib import Path - root = Path('dist') - notes = root / 'RELEASE-NOTES.md' - notes.write_text(notes.read_text() + '\n\nContainer ARMv7/ARM64 images are development snapshots: native incoming transit ownership is unresolved. Do not deploy them as mesh routers yet. See CONTAINER-INSTALL.md.\n') - files = sorted(p for p in root.iterdir() if p.is_file() and p.name != 'SHA256SUMS') - (root / 'SHA256SUMS').write_text(''.join(hashlib.sha256(p.read_bytes()).hexdigest() + ' ' + p.name + '\n' for p in files)) - PYTHON - cd dist - sha256sum -c SHA256SUMS - name: Save downloadable build bundle uses: https://github.com/actions/upload-artifact@v3 with: @@ -155,12 +168,6 @@ jobs: path: dist/ if-no-files-found: error retention-days: 30 - - name: Publish container manifest to registry - if: startsWith(github.ref, 'refs/tags/ros-') || inputs.publish == true || inputs.publish == 'true' - env: - QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} - QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} - run: tools/build-container.sh publish - name: Publish prerelease if: startsWith(github.ref, 'refs/tags/ros-') || inputs.publish == true || inputs.publish == 'true' env: diff --git a/.gitea/workflows/site.yml b/.gitea/workflows/site.yml new file mode 100644 index 0000000..a179b7f --- /dev/null +++ b/.gitea/workflows/site.yml @@ -0,0 +1,85 @@ +name: Publish and promote RouterOS project site +on: + push: + pull_request: +jobs: + site: + runs-on: mikrotik-olsr-builder + env: + PODMAN_URL: unix:///var/run/docker.sock + steps: + - uses: https://github.com/actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + - name: Validate site, feed, and chart + run: ./test-site.sh + - name: Build site image + env: + IMAGE_TAG: ${{ gitea.sha }} + run: podman build --file service/Dockerfile --tag "localhost/mikrotik-olsr-site:${IMAGE_TAG}" . + - name: Publish image and chart + if: gitea.ref == 'refs/heads/main' + env: + COMMIT_SHA: ${{ gitea.sha }} + PACKAGE_USERNAME: ${{ secrets.PACKAGE_USERNAME }} + PACKAGE_TOKEN: ${{ secrets.PACKAGE_TOKEN }} + QUAY_USERNAME: ${{ secrets.QUAY_USERNAME }} + QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} + run: | + set -euo pipefail + test -n "$PACKAGE_USERNAME" && test -n "$PACKAGE_TOKEN" + test -n "$QUAY_USERNAME" && test -n "$QUAY_TOKEN" + image_tag="sha-${COMMIT_SHA}" + image="registry.brunner.ninja/feedc0de/mikrotik-olsr-site:${image_tag}" + printf '%s' "$QUAY_TOKEN" | podman login registry.brunner.ninja \ + --username "$QUAY_USERNAME" --password-stdin + podman tag "localhost/mikrotik-olsr-site:${COMMIT_SHA}" "$image" + podman push "$image" + chart_version=$(./scripts/chart-version.sh) + mkdir -p .chart-packages + helm package chart/mikrotik-olsr-site --destination .chart-packages \ + --version "$chart_version" --app-version "$image_tag" + curl --fail --silent --show-error --user "${PACKAGE_USERNAME}:${PACKAGE_TOKEN}" \ + --request POST --upload-file ".chart-packages/mikrotik-olsr-site-${chart_version}.tgz" \ + https://code.brunner.ninja/api/packages/feedc0de/helm/api/charts + printf 'CHART_VERSION=%s\n' "$chart_version" >> "$GITHUB_ENV" + - name: Check current main + id: current-main + if: gitea.ref == 'refs/heads/main' + env: + COMMIT_SHA: ${{ gitea.sha }} + run: | + git fetch --no-tags origin main + if test "$(git rev-parse FETCH_HEAD)" = "$COMMIT_SHA"; then + echo 'promote=true' >> "$GITHUB_OUTPUT" + else + echo 'promote=false' >> "$GITHUB_OUTPUT" + fi + - uses: https://github.com/actions/checkout@v4 + if: steps.current-main.outputs.promote == 'true' + with: + repository: feedc0de/homelab-gitops + ref: main + token: ${{ secrets.GITOPS_TOKEN }} + path: homelab-gitops + - name: Promote chart to Argo CD + if: steps.current-main.outputs.promote == 'true' + working-directory: homelab-gitops + env: + GITOPS_TOKEN: ${{ secrets.GITOPS_TOKEN }} + run: | + set -euo pipefail + test -n "$GITOPS_TOKEN" + ../scripts/update-chart-version.sh mikrotik-olsr-site "$CHART_VERSION" + git config user.name feedc0de + git config user.email feedc0de@users.noreply.code.brunner.ninja + git add bootstrap/mikrotik-olsr-site.yaml + if git diff --cached --quiet; then exit 0; fi + git commit -m "Deploy mikrotik-olsr-site chart ${CHART_VERSION}" + for attempt in 1 2 3; do + git push origin HEAD:main && exit 0 + test "$attempt" = 3 && exit 1 + git fetch origin main + git rebase origin/main + done diff --git a/README.md b/README.md index fe72991..f226d02 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,18 @@ for MikroTik hardware. [Source](https://code.brunner.ninja/feedc0de/mikrotik-olsr) · [Prebuilt downloads](https://code.brunner.ninja/feedc0de/mikrotik-olsr/releases) · [Build instructions](docs/building.md) · [OLSR status](docs/olsr.md) · +[RouterOS internals](docs/routeros-internals.md) · +[update feed](docs/update-server.md) · +[platform analysis inputs](docs/platform-analysis.md) · [Installation](docs/INSTALL.md) · [CI releases](docs/ci.md) **Experimental.** Own-key installation, OLSRv1 neighbor discovery, hundreds of -active native dynamic routes, and enable/disable cleanup have been tested on a -hAP ac2. The embedded Winbox/WebFig configuration UI is still under development. -A successfully signed NPK is not evidence that its service works on a device. -See the release notes for the exact hardware and behavior tested in each build. +dynamic BGP routes in RouterOS, and enable/disable cleanup have been tested on a +hAP ac2. The current scheduler-backed UI is not a native OLSR subsystem, and +the project does not yet implement `/olsr` commands or native OLSR-protocol +routes. ARM64/AArch64 NPK builds are now included in the build path, but have +not yet been validated on hardware. See the release notes for the +exact hardware and behavior tested in each build. ## How it works @@ -37,7 +42,7 @@ The matching private signing keys stay on your build machine or in CI secrets. They are never included in firmware, the website, or public build artifacts. The Ethernet/VLAN package set consists of `routeros`, `option` for open-source -runtime utilities, and `olsr` for the daemon and service integration. Add a Wi-Fi +runtime utilities, and `olsr` for the daemon and experimental service wrapper. Add a Wi-Fi package only if you need wireless interfaces and the complete set fits. Package contents and versions must fit the target device. For example, a hAP ac2 has only 16 MB of flash; the complete set may not fit without reducing the system/runtime payload. Do not assume `netinstall -f` fixes this. @@ -69,6 +74,12 @@ The build scripts run on Linux (including an appropriate Linux VM). They produce both Linux and Windows Netinstall binaries. You do not need Windows to patch the Windows executable. +The public site and RouterOS metadata feed are built from `service/` and `site/`; +the Kubernetes chart is in `chart/mikrotik-olsr-site/`. The current feed serves +channel versions and changelogs, while package-download protocol support and the +RouterOS updater-host patch remain under investigation. See +[update-server.md](docs/update-server.md). + Install prerequisites on Debian/Ubuntu: ```sh @@ -142,8 +153,9 @@ python tools/routeros.py create \ ``` This constructs and signs the package; it does not invent a startup registration -or GUI backend. [OLSR implementation status](docs/olsr.md) describes what still -needs to be validated before this package is ready for normal installation. +or GUI backend. [RouterOS integration internals](docs/routeros-internals.md) +explains the package format, option payload, current service lifecycle, UI +descriptor limits, and the work needed for native `/olsr` and OLSR route objects. For the complete download/build sequence and custom payload handling, see [building.md](docs/building.md). For CI configuration, see [ci.md](docs/ci.md). @@ -184,8 +196,9 @@ hAP ac2, same-version replacement of our OLSR NPK worked and retained the router configuration. Reserve flash headroom and check `/log` and `/system/package` after reboot. -The current service can be enabled or disabled using the `OLSR` entry under -**System → Scheduler**, once configured. Its two-second heartbeat keeps a -supervisor alive; disabling it withdraws native routes and clears the reserved -kernel tables within approximately eight seconds in the measured test. +The current service can be enabled or disabled using the scheduler-backed OLSR +entry once configured. Its two-second heartbeat keeps a supervisor alive; +disabling it withdraws dynamic BGP routes and clears the reserved Linux kernel +tables within approximately eight seconds in the measured test. The scheduler +is a temporary boot/service mechanism, not the desired OLSR configuration UI. See [hardware results](docs/hardware-test.md) for limitations. diff --git a/chart/mikrotik-olsr-site/Chart.yaml b/chart/mikrotik-olsr-site/Chart.yaml new file mode 100644 index 0000000..ea3c64b --- /dev/null +++ b/chart/mikrotik-olsr-site/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +name: mikrotik-olsr-site +description: Public project site and RouterOS update feed for mikrotik-olsr +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://mikrotik.brunner.ninja +sources: + - https://code.brunner.ninja/feedc0de/mikrotik-olsr diff --git a/chart/mikrotik-olsr-site/templates/_helpers.tpl b/chart/mikrotik-olsr-site/templates/_helpers.tpl new file mode 100644 index 0000000..913c711 --- /dev/null +++ b/chart/mikrotik-olsr-site/templates/_helpers.tpl @@ -0,0 +1,19 @@ +{{- define "site.name" -}} +{{- .Chart.Name -}} +{{- end -}} + +{{- define "site.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{- define "site.labels" -}} +app.kubernetes.io/name: {{ include "site.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | quote }} +{{- end -}} diff --git a/chart/mikrotik-olsr-site/templates/deployment.yaml b/chart/mikrotik-olsr-site/templates/deployment.yaml new file mode 100644 index 0000000..222fcf9 --- /dev/null +++ b/chart/mikrotik-olsr-site/templates/deployment.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "site.fullname" . }} + labels: + {{- include "site.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "site.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "site.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 10001 + runAsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: site + image: {{ printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.service.port }} + protocol: TCP + startupProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 2 + failureThreshold: 30 + readinessProbe: + httpGet: + path: /healthz + port: http + livenessProbe: + httpGet: + path: /healthz + port: http + resources: + {{- toYaml .Values.resources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + terminationGracePeriodSeconds: 10 diff --git a/chart/mikrotik-olsr-site/templates/ingress.yaml b/chart/mikrotik-olsr-site/templates/ingress.yaml new file mode 100644 index 0000000..c839ba7 --- /dev/null +++ b/chart/mikrotik-olsr-site/templates/ingress.yaml @@ -0,0 +1,29 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "site.fullname" . }} + labels: + {{- include "site.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: {{ .Values.ingress.className | quote }} + {{- with .Values.ingress.tls }} + tls: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + - host: {{ .Values.ingress.host | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "site.fullname" . }} + port: + name: http +{{- end }} diff --git a/chart/mikrotik-olsr-site/templates/service.yaml b/chart/mikrotik-olsr-site/templates/service.yaml new file mode 100644 index 0000000..ddfb19f --- /dev/null +++ b/chart/mikrotik-olsr-site/templates/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "site.fullname" . }} + labels: + {{- include "site.labels" . | nindent 4 }} +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: {{ include "site.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + ports: + - name: http + port: {{ .Values.service.port }} + targetPort: http + protocol: TCP diff --git a/chart/mikrotik-olsr-site/values.schema.json b/chart/mikrotik-olsr-site/values.schema.json new file mode 100644 index 0000000..7ed0499 --- /dev/null +++ b/chart/mikrotik-olsr-site/values.schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "replicaCount": {"type": "integer", "minimum": 1}, + "image": { + "type": "object", + "required": ["repository", "pullPolicy"], + "properties": { + "repository": {"type": "string", "minLength": 1}, + "tag": {"type": "string"}, + "pullPolicy": {"type": "string", "enum": ["Always", "IfNotPresent", "Never"]} + } + }, + "service": {"type": "object", "properties": {"port": {"type": "integer", "minimum": 1, "maximum": 65535}}}, + "ingress": { + "type": "object", + "required": ["enabled", "host"], + "properties": { + "enabled": {"type": "boolean"}, + "className": {"type": "string"}, + "host": {"type": "string", "minLength": 1}, + "annotations": {"type": "object"}, + "tls": {"type": "array"} + } + }, + "resources": {"type": "object"} + } +} diff --git a/chart/mikrotik-olsr-site/values.yaml b/chart/mikrotik-olsr-site/values.yaml new file mode 100644 index 0000000..59dd789 --- /dev/null +++ b/chart/mikrotik-olsr-site/values.yaml @@ -0,0 +1,24 @@ +replicaCount: 1 + +image: + repository: registry.brunner.ninja/feedc0de/mikrotik-olsr-site + tag: "" + pullPolicy: IfNotPresent + +service: + port: 8080 + +ingress: + enabled: false + className: traefik + annotations: {} + host: mikrotik.example.invalid + tls: [] + +resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi diff --git a/docs/RELEASE-NOTES.md b/docs/RELEASE-NOTES.md index c129de1..e727ef2 100644 --- a/docs/RELEASE-NOTES.md +++ b/docs/RELEASE-NOTES.md @@ -1,54 +1,42 @@ -# First experimental release: RouterOS 7.25.1 / ARM +# Experimental RouterOS OLSR packages -This release is for experiments on spare hardware. It is not a finished native -OLSR subsystem. It includes own-key Linux and Windows Netinstall, a patched -RouterOS system package, an optional re-signed Wi-Fi package, a minimal option -runtime and an OLSRv1 package with a local BGP bridge. +The ARMv7 package set has been installed and tested on a hAP ac2 running +RouterOS 7.25.1. ARM64/AArch64 release bundles now build and pass signature +verification, including the matching `wifi-qcom` package. ARM64 hardware +installation and runtime behavior remain unvalidated. -## Tested on a hAP ac2 +## ARMv7 hardware results -- Own-key Netinstall and subsequent package-only updates booted successfully. -- The original RouterOS license remained at level 4. -- OLSRv1 discovered symmetric neighbors on a VLAN with a /32 node address. -- Around 700 active dynamic routes appeared in RouterOS through loopback BGP. -- Neighbor ping succeeded. Broader forwarding, throughput, failover and long-term - operation have not been validated. -- Disabling the OLSR scheduler removed native and private OLSR routes within - approximately eight seconds; re-enabling restored neighbor discovery/routes. -- Free flash with routeros + option + olsr was approximately 1.34 MiB. The Wi-Fi - package was not included in this measurement and is not a tested fit. +- Own-key Netinstall and package-only updates booted successfully. +- The original RouterOS level 4 license remained available. +- OLSRv1 discovered symmetric FunkFeuer neighbors on a VLAN with the assigned + `/32` node address. +- Around 700 active dynamic BGP routes appeared in RouterOS through the local + BIRD adapter. Direct-neighbor pings succeeded; broader forwarding, + throughput, failover, and long-term operation were not validated. +- Disabling the scheduler-backed service withdrew RouterOS routes and cleared + its private Linux tables in roughly eight seconds during the measured test. +- Free flash with RouterOS, option, and olsr was about 1.34 MiB on the tested + device. The Wi-Fi package was not included and may not fit. -## Native UI status +## Important limitations -An experimental OLSR > Service table provides native enable/disable controls in -Winbox/WebFig. It currently uses a scheduler heartbeat internally. It does not -restart olsrd on every heartbeat. Interface/address and accepted-prefix settings -are configured using configure-device.py for now. Native settings windows, HNA -advertisement controls, and native live logs are unfinished. HNA advertisements -are blocked in this release. Logs can be read using the setup tool and remain in -bounded RAM buffers. The separate development web page has been removed. +This is not yet a first-class RouterOS OLSR subsystem. The current service uses +a saved scheduler record for startup and enable/disable, so it appears in Jobs +and has a run count. Configuration is a file plus one service record. Routes +are dynamic BGP routes, not OLSR-protocol routes. HNA and default-route import +are rejected. There is no `/olsr` command tree, multiple persistent instances, +custom OLSR log topic, or detailed OLSR configuration window. -An earlier experimental UI descriptor caused Winbox crashes and is not included. -The reduced service descriptor appeared successfully on the test router. Native -UI compatibility across other Winbox versions is not established. +The embedded UI descriptor exposes only a scheduler-backed enable control and +is not validated across Winbox/WebFig versions. An earlier descriptor caused a +Winbox crash. Read [routeros-internals.md](routeros-internals.md) for verified +descriptor capabilities and the required backend work. -## Scope and limitations - -Only ARM on the tested hAP ac2 has hardware validation. cAP ac hardware is a -candidate, not a tested target. There is currently one OLSR interface. The default -accepted network is 10.0.0.0/8; additional explicitly selected IPv4 networks can be -configured. Default route import is excluded. BGP sessions remain on loopback and -export no RouterOS routes back to OLSR. Tables 201/202/203 and kernel protocol 100 -are reserved for OLSR; do not use them for other routing configuration. - -Normal stop and heartbeat-expiry cleanup were tested. Full crash/reboot/link-loss -coverage is incomplete. Unexpected supervisor termination can leave private -kernel routes until restart/reboot, although the BGP session withdrawal removes -the native routes. BIRD logs include a RouterOS address-notification warning; -neighbor discovery and route import worked despite it. Hardware offload and -routing-policy interactions have not been validated. - -Read INSTALL.md before installing. Use all packages from the same signing -keyset and RouterOS version. Do not force installation when Netinstall reports -insufficient space. The downloadable Wi-Fi package is optional, not an assurance -that all packages fit a 16 MiB device. +The package builder produces ARMv7 and ARM64/AArch64 targets. The ARM64 runtime +has now completed a cross-build and package-signature checks, but has not been +installed on ARM64 hardware. Use packages matching the RouterOS version, +architecture, and signing key installed on the device. Do not force +installation when Netinstall reports insufficient space. See +[INSTALL.md](INSTALL.md), [OLSR status](olsr.md), and +[hardware results](hardware-test.md). diff --git a/docs/building.md b/docs/building.md index c98b231..9da3412 100644 --- a/docs/building.md +++ b/docs/building.md @@ -24,27 +24,46 @@ what was used; hashes recorded after an HTTPS download are not an independent MikroTik signature on Netinstall. NPK inputs additionally undergo Ed25519 checking. The tools retain package version and architecture metadata. Do not mix different -RouterOS versions, architectures, or publishers' keys. The system build currently -has an exercised ARM path; other architectures need their own integration tests. +RouterOS versions, architectures, or publishers' keys. ARMv7 and ARM64/AArch64 +NPK generation are now available in the build scripts; only ARMv7 has completed +hardware acceptance. ARM64 output must be treated as experimental until it has +been installed and tested on an ARM64 router. The release builder has also been +locally verified against RouterOS 7.23.8 ARM and 7.24.6 ARM/ARM64 inputs. ## Build option and OLSR payloads ```sh -tools/build-runtime.sh +tools/build-runtime.sh arm python tools/build-release.py --version 7.25.1 --arch arm \ --wifi wifi-qcom-ac --keys .keys/local.json --output dist-with-runtime \ - --option-root build/option-root --olsr-root build/olsr-root + --option-root build/runtime-arm/option-root --olsr-root build/runtime-arm/olsr-root ``` -The runtime script downloads checksum-pinned ARM musl cross tools and BusyBox -source, and checks out a pinned OLSRv1 source revision. It builds a minimal static -BusyBox and the OLSR daemon. It never downloads `option.npk` from another firmware -publisher. You can instead provide your own audited payload directories. +The runtime script downloads checksum-pinned ARM or ARM64 musl cross tools and +BusyBox source, and checks out a pinned OLSRv1 source revision. It builds a +minimal static BusyBox and the OLSR daemon. It never downloads `option.npk` from +another firmware publisher. For AArch64 packages, stage the AArch64 payloads and +use the matching `arm64` system/add-on templates: -The resulting `option` package contains utilities, not the other project's -license generator or shell-unlock implementation. The resulting `olsr` package -contains the daemon development payload. Automatic startup, route cleanup, and -UI integration remain separate work described in [olsr.md](olsr.md). +```sh +tools/build-runtime.sh arm64 +python tools/build-release.py --version 7.25.1 --arch arm64 \ + --keys .keys/local.json --output dist-arm64 \ + --option-root build/runtime-arm64/option-root --olsr-root build/runtime-arm64/olsr-root +``` + +Select `--wifi wifi-qcom` only for compatible ARM64 hardware. `wifi-qcom-ac` is +the ARMv7 package. Omit Wi-Fi when the device does not need it, especially on +small-storage hardware. You can also provide your own audited payload +directories instead of running the runtime cross-build. + +The resulting `option` package contains BusyBox utilities, not another +publisher's license generator or shell-unlock implementation. The resulting +`olsr` package contains the daemon and experimental service payload. Its current +startup uses a persistent RouterOS scheduler object and its route adapter uses +BGP; it does not create native `/olsr` command handlers. See +[routeros-internals.md](routeros-internals.md) for package anatomy, UI limits, +and the distinction between this adapter and the intended native subsystem. The cross compiler itself is a prebuilt, checksum-pinned toolchain. This is not a claim that every build tool has been bootstrapped from independently audited source. diff --git a/docs/ci.md b/docs/ci.md index c365b26..62e11fb 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -46,26 +46,24 @@ exit. Only the public key export is included in the release output. ## Run a release -Push a new annotated tag matching `ros-VERSION-arm-rNUMBER` to run the complete -build and publish automatically, for example `ros-7.25.1-arm-r2`. The version is -parsed from the tag. Tags must point to reviewed release source; protect release -tags as well as main because tagged workflows can access signing credentials. +Push a new annotated tag matching `ros-VERSION-arm-rNUMBER` or +`ros-VERSION-arm64-rNUMBER` to build and publish automatically, for example +`ros-7.25.1-arm64-r1`. Tagged releases include the architecture's optional +Wi-Fi package. The version is parsed from the tag. Tags must point to reviewed +release source; protect release tags as well as main because tagged workflows +can access signing credentials. -Alternatively run **Build NPK and container release** manually on `main`, selecting the RouterOS +Alternatively run **Build RouterOS package release** manually on `main`, selecting the RouterOS version and a new tag such as `ros-7.25.1-arm-r1`. The workflow builds BusyBox, olsrd, BIRD and the service helpers, patches/signs packages and both installers, verifies signatures/checksums, and saves a downloadable Actions artifact. -The same run also builds ARMv7 and ARM64 container archives. These are included -in the release assets together with container instructions and the interface -filter generator. Containers are development snapshots pending native forwarding -acceptance; a successful build is not a hardware certification. +The release workflow only builds RouterOS packages and patched Netinstall. The +separate container workflow builds and publishes ARMv7/ARM64 images on `main`. -Enable the `publish` input to push the multiarchitecture container manifest tagged -with the full source commit, then create a Gitea prerelease. A registry push -failure prevents release publication (a successful registry push cannot be -rolled back if a later release upload fails). With publication disabled, -no publication credentials are needed. The artifact step uses upload-artifact v3 for -Gitea compatibility. A working runner and signing secret are still required. +Enable the `publish` input to create a Gitea prerelease. With publication +disabled, CI saves a downloadable Actions artifact but does not create a Gitea +release. The artifact step uses upload-artifact v3 for Gitea compatibility. A +working runner and signing secret are still required. The uploader first creates a draft, uploads all checksummed files, and only then publishes it. A failed upload leaves a draft for inspection. It never overwrites @@ -78,11 +76,18 @@ prerelease description explicitly records the unfinished OLSR integration. ## Download page -The buildless page in `site/` fetches public release metadata from the Gitea API. -Host that directory on your static web server. The server must allow browser -access to the public Gitea API (CORS for a different origin). The page also links -directly to Gitea releases when API access is unavailable. No private token belongs -in the page. +The service in `service/` hosts the static page and a same-origin proxy for public +Gitea release metadata. Its portable chart is in +`chart/mikrotik-olsr-site/`; the homelab values use +`https://mikrotik.brunner.ninja`. The service also implements the verified +plain-text RouterOS channel and changelog responses. Package download support +and the updater host patch are not yet verified; see +[update-server.md](update-server.md). + +The service chart workflow publishes an immutable image and chart from `main`, +then promotes the chart version to Argo CD. It needs `PACKAGE_USERNAME`, +`PACKAGE_TOKEN`, `QUAY_USERNAME`, `QUAY_TOKEN`, and `GITOPS_TOKEN`; it receives no +Kubernetes credentials. ## Manual first release diff --git a/docs/olsr.md b/docs/olsr.md index a028e0e..1a57b1c 100644 --- a/docs/olsr.md +++ b/docs/olsr.md @@ -1,57 +1,114 @@ # OLSR implementation status -The target is OLSRv1 (`olsrd`), with an OLSR enable control, editable configuration, -neighbor status, and automatic route withdrawal when disabled. +The target is an OLSRv1 service integrated with RouterOS configuration, routing, +Winbox, and WebFig. The current package is an experimental daemon wrapper and +does not yet meet that target. -## What has been established +## Verified behavior -- The development device is a hAP ac2, ARMv7/IPQ4019, with 128 MB RAM and 16 MB flash. -- OLSRv1 link-quality HELLO/TC traffic is present on the development VLAN. -- Linux routes added externally persist, but were not displayed as native - RouterOS routes in the initial test. Forwarding and RouterOS route presentation - are separate integration requirements. -- A static musl ARM daemon can be built at roughly 431 KB before packaging. -- Own-key RouterOS, option and olsr packages installed successfully on the hAP ac2. -- The original Level 4 license is recognized again, and the official login - program starts the source-built BusyBox shell through its bash entry point. -- The installed OLSR binary executes successfully. A 90-second live test formed - symmetric ETX links to two FunkFeuer neighbors and learned routes into table - 201, with learned default routes isolated in table 202. -- Three pings to a directly connected OLSR neighbor succeeded using a targeted - policy rule. OLSR removed its routes on SIGTERM; both test tables were empty - afterward. The temporary policy rule was also removed. -- This installation reports 1768 KiB free flash, without Wi-Fi packages. +- On a hAP ac2 (ARMv7, 128 MiB RAM, 16 MiB flash), custom-key RouterOS, + `option`, and `olsr` packages installed and retained the original level 4 + license. +- OLSRv1 formed symmetric ETX links to two FunkFeuer neighbors on a VLAN using + the assigned `/32` node address. +- OLSRd installed routes into Linux kernel table 201. BIRD exported accepted + routes to RouterOS over a local BGP session; around 700 active dynamic BGP + routes were visible in RouterOS and direct-neighbor pings succeeded. +- Disabling the scheduler-backed service withdrew the RouterOS BGP routes and + cleared the private Linux tables in about eight seconds in the measured test. +- OLSRd and BIRD logs are bounded to 32 KiB RAM rings. The package-only update + path worked on the tested installation. -## Still under development +These results validate the BGP adapter design on one ARMv7 device. They do not +validate arbitrary forwarding, hardware offload, multi-instance operation, +ARM64 NPKs, or a native OLSR routing protocol in RouterOS. -- Persistent configuration and interface selection. -- Supervised startup, checkbox control, bounded logs, and crash cleanup. -- Native RouterOS route visibility. Kernel routes were usable but were absent - from RouterOS's route database. A local BGP bridge is being evaluated; it is - not yet built into a validated release. -- Editable WebFig integration and native Winbox integration. A custom WebFig - descriptor alone does not register a native Winbox menu. -- Reboot, forwarding, upgrade, and installation testing of the full service. +## Current runtime model -The upstream example is a read-only WebFig menu. Its private loader patch is -not needed for the shell and OLSR execution established in our hardware test. +`configure-device.py` verifies a RouterOS interface and node address, writes one +OLSRd configuration file, creates BGP filters/connection objects, and creates a +persistent `/system scheduler` heartbeat. The heartbeat runs at startup and +every two seconds. A detached supervisor starts OLSRd and BIRD; disabling the +scheduler expires a lease, then the supervisor cleans up routes it owns. -## Required service behavior +OLSRd uses the Linux interface carrying the configured `MainIp`. RouterOS +interface names are not assumed to equal Linux names. The runtime currently +uses fixed Linux routing tables 201, 202, and 203 and protocol number 100. It +does not map arbitrary RouterOS routing-table names to Linux table IDs. -The eventual service must run only on explicitly configured interfaces. RouterOS -interface names differ from Linux names and cannot be hard-coded: for example, -a RouterOS VLAN can appear as `vlan8` in Linux regardless of its display name or -VLAN ID. +For native RouterOS visibility, BIRD advertises approved routes to RouterOS over +local eBGP. RouterOS therefore labels them as BGP. This is intentional in the +current adapter, but it is not the desired final result. The target is dynamic +routes owned by OLSR, labeled OLSR, with no BGP connection or BIRD process. No +supported public RouterOS custom route-protocol API has been identified yet; +see [RouterOS internals](routeros-internals.md) for the constraint and research +plan. -Disabling OLSR must stop the daemon gracefully and remove only routes/rules owned -by the service. Crash cleanup must not depend solely on the daemon's graceful -shutdown path. A dedicated routing protocol/table or explicit ownership records -are needed; flushing all routes is unacceptable. +## Current limits -Neighbor discovery must use the community network's assigned address and -compatible link-quality settings. Do not silently advertise a default route, -management subnet, or unrelated LAN through HNA. +- One OLSR instance and one configured interface in the NPK tooling. +- No persistent `/olsr` configuration objects or `/olsr/print`, `/olsr/add`, + `/olsr/set`, `/olsr/enable`, or `/olsr/remove` commands. +- The service record is a scheduler object, so it appears as a job and its run + count increases. This is a boot hook, not a first-class OLSR subsystem. +- Interface and accepted-prefix configuration is written through the setup + script, not an OLSR editor window. +- HNA advertisement is rejected by the runtime. Default-route import is + explicitly rejected by BIRD and the RouterOS filter generator. +- The Winbox/WebFig descriptor exposes only a scheduler-backed enable control. + It does not provide interface, HNA, routing-table, neighbor, or per-instance + log views. The descriptor is experimental and not validated across clients. +- The bounded RAM logs are not native RouterOS log records and have no custom + `olsr` topic or instance association. +- ARM64/AArch64 NPK and runtime builds are now wired into the release pipeline, + but have not yet passed a complete local build or device test. The separate + container workflow's ARM64 image is a distinct deployment path. -The `build-olsrd.sh` output is a development artifact, not a promise of a working -RouterOS service. Release notes must distinguish package-signature validation, -execution tests, protocol tests, UI tests, and actual hardware installation. +## Target first-class object model + +The intended subsystem should store one persistent RouterOS object per OLSR +instance. Its schema should cover: + +- instance `name`, enabled state, and service status; +- one or more RouterOS interface names, each with its assigned source IP; +- OLSRv1 link-quality and timing parameters; +- explicitly allowed HNA prefixes, with default-route advertisement opt-in; +- the RouterOS routing table that receives learned routes; +- bounded per-instance log settings and live status. + +The main list should show enabled state, name, interfaces, routing table, +neighbor count, route count, and last status. A detail view should provide +General, Interfaces, HNA, Routing, Neighbors, Routes, and Logs tabs. The service +manager should start/stop instances from the persistent object database, restore +enabled instances at boot, validate every referenced interface/table, and +withdraw only the routes and rules owned by that instance. + +This model requires real RouterOS configuration handlers. A UI descriptor by +itself cannot create the `/olsr` command tree or its persistent object schema. +The current scheduler row should not be presented as the finished UI. + +## Work required before claiming first-class support + +1. Identify a supported or verifiable RouterOS extension point for persistent + configuration objects, command registration, service startup, log topics, + and dynamic route ownership. The package writer and descriptor generator do + not provide these handlers. +2. Determine whether RouterOS has an internal route injection API that allows + an OLSR protocol owner to create dynamic routes without BGP. Validate it on a + disposable device before replacing the tested adapter. +3. Implement interface and routing-table mapping at the RouterOS boundary; + Linux device names and table numbers are not stable substitutes for RouterOS + object names. +4. Add multi-instance supervision, crash/reboot cleanup, and per-instance log + rings without routine flash writes. +5. Implement HNA as explicit, validated prefixes. Require a deliberate choice + before allowing `0.0.0.0/0`; do not import or advertise a default implicitly. +6. Build and test an actual native UI against the matching backend, then test + Winbox and WebFig across supported client versions. Remove the scheduler + presentation only when the replacement has a real object backend. +7. Test route withdrawal, asymmetric link failure, policy routing, throughput, + hardware-offload state, low-flash updates, and ARM64 separately. + +Until these items are complete, releases must describe the current BGP adapter +and scheduler-backed UI accurately. A valid signature or successful neighbor +discovery is not evidence that the first-class integration is implemented. diff --git a/docs/platform-analysis.md b/docs/platform-analysis.md new file mode 100644 index 0000000..76babda --- /dev/null +++ b/docs/platform-analysis.md @@ -0,0 +1,34 @@ +# RouterOS platform analysis inputs + +The ignored `build/official-platforms-7.25.1/` directory contains official +7.25.1 system NPKs and `all_packages` archives downloaded for comparative +analysis. It is deliberately not committed: the archives are large and can be +re-fetched from MikroTik. `SHA256SUMS` records the exact local downloads. + +The system NPK Ed25519 signatures were verified against the official v7 key by +the repository's NPK parser for ARM, ARM64, MIPSBE, MMIPS, PowerPC, SMIPS, TILE, +and x86. NPK verification is not a substitute for checking the source URL and +release version. The package patcher currently supports ARM, ARM64, x86, +MIPSBE, MMIPS, SMIPS, and PowerPC; TILE is analysis-only in the tool interface. +The OLSR runtime release builder currently targets ARM and ARM64. + +The system filesystems were also extracted to the ignored +`build/platform-roots-7.25.1/` directory. This comparison found that ARM64's +`nova/bin/sys2` is still a 32-bit ARM ELF, while MIPSBE, MMIPS, PowerPC, SMIPS, +TILE, and x86 each contain binaries for their own CPU. This makes other +platforms useful for comparing common RouterOS service behavior without +assuming the binaries are byte-identical. + +For the 7.25.1 ARM build, `upgrade.mikrotik.com` appears in `sys2`, and the +updater references strings for `NEWESTa`, `/CHANGELOG`, `/packages.csv`, and +`/routeros/%s?version=%s`. The public channel and changelog responses are now +verified. The package-list route still needs Ghidra reference analysis or +captured device traffic before it is implemented by the custom feed. See +[update-server.md](update-server.md). + +Example local verification: + +```sh +cd build/official-platforms-7.25.1 +sha256sum -c SHA256SUMS +``` diff --git a/docs/routeros-internals.md b/docs/routeros-internals.md new file mode 100644 index 0000000..36cfd5e --- /dev/null +++ b/docs/routeros-internals.md @@ -0,0 +1,266 @@ +# RouterOS package and integration internals + +This document records what the current project does, what the RouterOS package +and UI formats appear to permit, and which parts still require reverse +engineering. The distinction matters: changing and signing an NPK does not add a +new RouterOS subsystem or command handler by itself. + +## Current implementation at a glance + +| Area | Current behavior | +| --- | --- | +| System trust | `tools/routeros.py npk --patch-system` replaces the package verification key in the system filesystem and kernel, then signs the modified system NPK. | +| `option` NPK | A small statically linked BusyBox build and applet links, including a `bash`-named entry point. It contains no license generator or shell-unlock patch. | +| `olsr` NPK | ARMv7 or ARM64 OLSRd, BIRD, startup/supervision helpers, one configuration file, and a small experimental WebFig/Winbox descriptor. | +| Persistent service record | A RouterOS `/system scheduler` record named OLSR pulses the service and runs at startup. It is a real saved RouterOS config object, but it is not an OLSR subsystem. | +| OLSR configuration | A file under RouterOS Files, presented to the package as `/rw/disk/olsrd.conf`. The current tool configures one instance. | +| Route integration | OLSRd writes to private Linux kernel tables; BIRD exports accepted routes to RouterOS over a local eBGP session. RouterOS displays them as dynamic BGP routes. | +| Logs | OLSRd and BIRD output is kept in bounded RAM ring files. It is not yet a RouterOS `olsr` log topic or a per-instance native log window. | +| ARM64 NPKs | A complete ARM64 release bundle now builds and verifies locally. RouterOS hardware acceptance is still pending. Container images use a separate pipeline. | + +The running development package therefore does **not** yet provide `/olsr`, +native OLSR route objects, multiple persistent instances, HNA editing, or an +independent OLSR configuration window. The BGP bridge is an adapter to the +RouterOS routing database, not part of OLSR itself. + +## NPK construction + +An NPK is a structured Nova package, not just a compressed directory. It +contains package identity and version metadata, package payload sections, and +signature material. The current tooling uses the `NovaPackage` reader/writer in +`vendor/mikrotikpatch/npk.py` and preserves metadata from a matching official +add-on template. + +For a new add-on, the build flow is: + +1. Download the official package archive for the same RouterOS release and + architecture. Verify the official NPK signature before using a package as a + template. +2. Select a small add-on template such as `gps`. The template supplies valid + version, architecture, and package metadata. +3. Replace its package name and description, remove the template's file + container and install/uninstall scripts, and insert a SquashFS made from the + new package root directory. +4. Sign the resulting package with the project's Ed25519 and KCDSA private + keys. Reload the written NPK and verify both signatures before publishing. + +For example: + +```sh +python tools/routeros.py create \ + --template build/inputs-7.25.1-arm/gps-7.25.1-arm.npk \ + --root build/olsr-root --name olsr --description 'OLSRv1 routing service' \ + --keys .keys/local.json --output dist/olsr-7.25.1-arm.npk +``` + +This creates a signed package container. It does not tell RouterOS how to start +the program, persist OLSR objects, add a command tree, or create a Winbox +backend. Those behaviors must be implemented separately. + +The initial own-key install has a separate system-package step. The system NPK +contains verification keys in its filesystem and boot components. The patcher +checks the official signature, replaces the verification key in both places, +rebuilds the filesystem, and signs the result. An add-on re-sign alone cannot +make a custom key trusted by an unmodified system. The matching private keys +must remain outside NPKs and public release artifacts. + +## What the `option` package contains + +`tools/build-runtime.sh` cross-compiles BusyBox 1.37.0 using the pinned ARM +musl toolchain. It enables only the applets needed by the runtime and creates +links such as `bash`, `sh`, `ip`, `grep`, `sed`, `sleep`, and `kill` under +`/pckg/option/bin`. The package also includes its BusyBox configuration and +licenses. It does not contain the OLSR daemon; `olsrd` and service helpers belong +to the separate `olsr` package. + +The `bash` path is useful because the official login path on the tested image +expects a shell at that name. The observed `devel` SSH session consequently +gets a BusyBox shell. The `option` NPK itself does not grant credentials, +change user permissions, or raise a user's privilege. The modified system +trusts the project's package key; RouterOS license validation is a separate +mechanism and is not changed by the BusyBox package. + +The BusyBox applet list is deliberately small to conserve flash on 16 MiB +devices. Review the generated `build/option-root/share/busybox.config` when +adding an applet. Do not add a general-purpose tool or shell feature without +measuring the compressed package size and installed flash headroom on each +target. + +## Building OLSR for ARM + +The NPK runtime supports 32-bit ARM (`arm`, ARM EABI) and is being extended to +64-bit ARM (`arm64`, AArch64). The script pins an OLSRd source revision, applies +the scanner build adjustment, compiles OLSRd statically with the matching cross +toolchain, strips the binary, and stages it under the architecture-specific +`build/runtime-{arch}/olsr-root/bin/olsrd`. It then stages `olsrctl`, the supervisor launcher, +the child-process guard, the bounded log helper, configuration templates, BIRD, +and license files. `tools/routeros.py create` packages that staging tree. + +The cross compiler is a pinned prebuilt toolchain, not a fully bootstrapped +compiler. Source revisions, toolchain hashes, package hashes, and release +metadata should be recorded together. A signed NPK proves package integrity +under the selected key; it does not prove protocol behavior or RouterOS +integration. + +## RouterOS configuration and service lifecycle + +Winbox, WebFig, the console, and RouterOS API/REST are front ends to RouterOS +configuration handlers. Adding a supported RouterOS object through one front +end persists that object in RouterOS configuration; the other front ends then +read or edit the same object. A package filesystem is separate from this +configuration database. Merely adding files under `/pckg/olsr` does not create +an `/olsr` menu or a persistent OLSR instance. + +The current service uses the supported `/system scheduler` object as its +persistent boot hook. The setup tool creates a two-second heartbeat with +`start-time=startup`. At startup and every interval it invokes +`/system/shell`; `start-service` detaches the supervisor, which reads the +configuration file, starts OLSRd and BIRD, and owns cleanup. Disabling the +scheduler expires the service lease. This is why the current entry appears +under Jobs/Scheduler and why the run count changes: it is an implementation +mechanism, not a native OLSR service object. A scheduler record survives reboot; +the transient processes do not. Startup reconstructs the processes from that +record and the saved configuration file. + +### What a real `/olsr` command tree would require + +Commands such as `/olsr/print`, `/olsr/add name=funkfeuer`, `/olsr/set`, +`/olsr/enable`, and `/olsr/remove` require a RouterOS command/configuration +handler that knows the OLSR object schema and lifecycle. An NPK can install +files and package scripts, but the current public package build path exposes no +documented API for registering a new native command namespace or route +protocol. A shell executable can be called explicitly through `/system/shell` +or a script; that does not register `/olsr` in the RouterOS parser. + +Likewise, a service object needs schema validation, stable IDs, persistent +storage, update semantics, startup reconstruction, and clean removal. A +reasonable target object would include `name`, `disabled`, `interfaces`, +`main-ip`, accepted prefixes, HNA networks, routing table, link-quality +settings, and per-instance logging. RouterOS interface and routing-table names +must be resolved at the RouterOS boundary. Linux interface names, ifindexes, +and kernel table numbers are implementation details and must not be assumed to +match RouterOS names or table IDs. + +The current implementation does only a partial mapping: the service verifies +that the configured RouterOS interface and IP exist, then locates the Linux +interface carrying `MainIp`. It uses fixed Linux route tables 201, 202, and +203. It does not map arbitrary RouterOS routing-table names into Linux route +tables. + +## Native routes versus the current BGP bridge + +OLSRd installs OLSR routes in the Linux network namespace. On the tested +RouterOS image, those routes were usable by Linux but did not appear in +RouterOS's route database. The current ARM package runs BIRD as an adapter: +it imports only routes owned by OLSRd from private table 201 and advertises +approved prefixes to a local RouterOS BGP connection. RouterOS then owns +dynamic BGP route objects and resolves each gateway through the real OLSR +neighbor interface. Filters reject default-route import and prevent learned +routes from being sent back to OLSR. + +That design has useful native route visibility and withdraws routes with the +BGP session, but the protocol label is BGP. It does not meet the desired goal +of routes labeled OLSR with no BGP process or BGP configuration. RouterOS's +public interface does not currently document a third-party API for inserting +dynamic routes under a custom protocol. Adding static routes through RouterOS +REST/CLI would not be equivalent: they would be static RouterOS routes, not +routes owned by OLSR, and cleanup would need to track every route explicitly. + +The no-BGP design therefore needs one of these research results before it can +be implemented safely: + +- a supported RouterOS routing-protocol plugin or route-injection API; +- a carefully versioned internal RouterOS protocol-handler interface found + through binary analysis and validated on disposable hardware; or +- acceptance that RouterOS will show a different built-in protocol type. + +Directly writing RouterOS's internal route database or changing its private +protocol handler without a verified interface risks corrupting routing state +and is not a viable first implementation. + +## Winbox and WebFig descriptor format + +The experimental descriptor builder writes an object-literal `.jg` resource +and an `.info` index into `home/web/webfig` in the NPK filesystem. It gzip +compresses the descriptor, computes a CRC and compressed size, and gives the +resource a content-derived name. This is a private RouterOS UI resource format, +not a public extension SDK. The descriptor references existing RouterOS page +IDs, field IDs, and handlers. Defining a visible field does not create storage +or a backend command for it. + +Inspection of RouterOS 7.25.1's bundled descriptors shows constructs including +menus/items, maps and tables, fields, tabs, conditional groups, lookup/dynamic +values, toolbars, context-menu actions, and built-in actions. Official +descriptors use context-menu actions to open named existing windows/tabs and +`doit` actions to invoke existing commands. That demonstrates the UI can +compose a rich presentation over RouterOS handlers. It does not demonstrate +that a third-party NPK can register new handlers or arbitrary server-side +operations. + +The project has only exercised a small list/map descriptor with a name and +enable control, mapped to an existing scheduler page and its field IDs. A +previous descriptor crashed Winbox because it evaluated a row predicate where +no row existed. There is no compatibility guarantee across Winbox versions. +Context menus and tabs are present in official descriptor resources, but custom +tree widgets, arbitrary subwindows, arbitrary action-bar placement, and custom +data operations have not been validated as a third-party extension contract. + +The desired OLSR UI should eventually provide a service list with enabled state, +name, interfaces, route table, HNA count, neighbors, and route count; a separate +edit view with General, Interfaces, HNA, Routing, and Logs tabs; and a bounded +per-instance log view. It must be backed by real OLSR objects and handlers. The +current scheduler-backed descriptor cannot safely provide those controls, so +the UI should not be described as first-class until the backend exists. + +## Logs + +The current package starts one `log-ring` process for OLSRd and one for BIRD. +Each keeps the newest 32 KiB in RAM and atomically writes a snapshot under +`/ram/olsr`. The CLI setup tool asks the service to print those snapshots. The +package can copy snapshots into RouterOS Files on request, provided Files is +RAM-backed on the target. This avoids routine flash writes. + +These files are not RouterOS's central log records: there is no registered +`olsr` topic, native log row, instance ID, or live Winbox view. A native log +category and per-instance table require a handler that can retrieve bounded +log records and associate them with persistent OLSR objects. Until then, the +accurate interface is the existing `olsrctl logs` helper, not the standard +RouterOS Log window. + +## Firmware update endpoint + +The NPK patcher currently changes package-verification keys only. It does not +change RouterOS's update host, version feed, download URL, or channel metadata. +In the extracted RouterOS 7.25.1 ARM system package, `nova/bin/sys2` contains +the string `upgrade.mikrotik.com`. A string replacement alone is not enough to +make updates work: the expected version response, architecture and channel +selection, package URL, checksum/signature validation, and release asset layout +must all be understood and served compatibly. RouterOS must still reject any +asset not signed by the installed custom key. + +Ghidra analysis of the update client is now part of the reverse-engineering +work. The project should first document the request/response protocol and test +it against a local mock server. Only then should an optional system-patch mode +redirect the update client to a configurable project endpoint. The normal +MikroTik update path should not be silently rewritten, and update checks must +never install a package that fails the custom-key signature check. + +## Reverse-engineering workspace + +Ghidra is installed at `/opt/ghidra` in the current development environment. +There is no Ghidra MCP connector enabled in this session, but headless Ghidra +and the local GUI are available. Useful starting points in the extracted +ARMv7 RouterOS system are: + +- `nova/bin/sys2` for package/version update behavior; +- `nova/bin/parser` and `nova/lib/console/*.mem` for command registration and + command metadata; +- `nova/bin/route` and `nova/bin/moduler` for routing processes and module + loading; +- `home/web/webfig/*.jg.gz` and `.info` for the private UI resource format. + +The source NPKs are kept in the ignored `build/` analysis tree. Record each +input's SHA-256, RouterOS version, architecture, and extracted path in research +notes. Compare ARM and ARM64 binaries before assuming a structure or address is +portable. Keep analysis projects and patched test packages out of normal release +artifacts until the behavior is understood and reviewed. diff --git a/docs/update-server.md b/docs/update-server.md new file mode 100644 index 0000000..01aa261 --- /dev/null +++ b/docs/update-server.md @@ -0,0 +1,63 @@ +# RouterOS update-feed service + +The project site now includes a small HTTP service for the RouterOS v7 update +metadata used by the package updater. It serves the static landing page and +Gitea release list, plus the plain-text update responses verified against +MikroTik's public server. + +## Verified HTTP responses + +RouterOS requests channel metadata in this form: + +```http +GET /routeros/NEWESTa7.stable?version=7.23.8 +``` + +The response is plain text containing the offered version and Unix build time: + +```text +7.25.1 1793623193 +``` + +The service currently maps `testing` to 7.23.8 for the upgrade-path experiment, +`long-term` to 7.24.6, and `stable` to 7.25.1. The version query parameter is +accepted and ignored, as it is by the upstream endpoint. Unknown channels return +HTTP 200 with `No version available in this channel`, matching the updater's +plain-text response handling. + +After receiving a version, the RouterOS update UI requests its changelog: + +```http +GET /routeros/7.25.1/CHANGELOG +``` + +The body is the release's plain-text changelog. The three initial changelogs are +stored under `site/feed/changelogs/`; update them alongside the channel metadata +when publishing a new package version. `/healthz` is used by the Kubernetes +probes. `/api/releases` returns public Gitea release metadata for the landing +page. + +These endpoints are a working metadata feed, not yet a complete package updater. +The project has not yet changed `sys2` to query this service, and the package +download request and response format has not been confirmed. A version appearing +in the update dialog must not be treated as proof that the signed NPK download +will work. The test plan is to first patch the updater host in a test system NPK, +capture its requests on a disposable RouterOS device, implement the matching +package response and downloads, then test install and rollback before marking +updates available by default. + +## Hosting + +The service runs from `service/server.py`; the corresponding container and +portable Helm chart live in this repository. Its public deployment target is +`https://mikrotik.brunner.ninja`. The service has no credentials or writable +data. It reads the checked-in feed/changelog files and queries the public Gitea +release API for the download page. + +Local checks: + +```sh +python3 -m unittest service.test_server -v +podman build -f service/Dockerfile -t localhost/mikrotik-olsr-site:test . +helm lint chart/mikrotik-olsr-site +``` diff --git a/scripts/chart-version.sh b/scripts/chart-version.sh new file mode 100755 index 0000000..072ba9a --- /dev/null +++ b/scripts/chart-version.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +description=$(git -C "$ROOT" describe --tags --long --match 'site-v[0-9]*' --dirty) +case "$description" in + *-dirty) echo 'Refusing to package a dirty site chart' >&2; exit 1 ;; +esac +version=${description#site-v} +case "$version" in + *-0-g*) version=${version%%-0-g*} ;; +esac +printf '%s\n' "$version" diff --git a/service/Dockerfile b/service/Dockerfile new file mode 100644 index 0000000..c42b70b --- /dev/null +++ b/service/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.13-alpine +WORKDIR /app +COPY service/server.py /app/server.py +COPY site /app/site +RUN addgroup -S -g 10001 app && adduser -S -D -H -u 10001 -G app app +USER 10001:10001 +EXPOSE 8080 +ENTRYPOINT ["python3", "/app/server.py", "8080"] diff --git a/service/server.py b/service/server.py new file mode 100644 index 0000000..546ef56 --- /dev/null +++ b/service/server.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Serve the project site and the small plain-text RouterOS update feed.""" +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import sys +from urllib.error import URLError +from urllib.request import Request, urlopen +from urllib.parse import urlsplit + + +HERE = Path(__file__).resolve().parent +SITE = HERE / "site" +if not SITE.is_dir(): # Running from the source tree: service/server.py. + SITE = HERE.parent / "site" +FEED = json.loads((SITE / "feed" / "channels.json").read_text()) +RELEASES_URL = ( + "https://code.brunner.ninja/api/v1/repos/feedc0de/mikrotik-olsr/releases?limit=10" +) + + +class Handler(SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super().__init__(*args, directory=str(SITE), **kwargs) + + def send_bytes(self, status, body, content_type): + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "public, max-age=300") + self.end_headers() + if self.command != "HEAD": + self.wfile.write(body) + + def do_HEAD(self): + self.do_GET() + + def do_GET(self): + path = urlsplit(self.path).path + if path == "/healthz": + return self.send_bytes(200, b"ok\n", "text/plain; charset=utf-8") + if path == "/api/releases": + try: + req = Request(RELEASES_URL, headers={"User-Agent": "mikrotik-olsr-site"}) + with urlopen(req, timeout=8) as response: + body = response.read() + json.loads(body) + return self.send_bytes(200, body, "application/json; charset=utf-8") + except (URLError, TimeoutError, ValueError) as exc: + return self.send_bytes(502, (str(exc) + "\n").encode(), "text/plain; charset=utf-8") + + prefix = "/routeros/NEWESTa7." + if path.startswith(prefix): + channel = path[len(prefix):] + item = FEED.get(channel) + if item is None: + return self.send_bytes(200, b"No version available in this channel\n", "text/plain; charset=utf-8") + body = f"{item['version']} {item['epoch']}\n".encode("ascii") + return self.send_bytes(200, body, "text/plain; charset=utf-8") + + marker = "/CHANGELOG" + if path.startswith("/routeros/") and path.endswith(marker): + version = path[len("/routeros/"):-len(marker)].strip("/") + if version and all(ch.isdigit() or ch == "." for ch in version): + changelog = SITE / "feed" / "changelogs" / f"{version}.txt" + if changelog.is_file(): + return self.send_bytes(200, changelog.read_bytes(), "text/plain; charset=utf-8") + return self.send_bytes(404, b"Unknown RouterOS changelog\n", "text/plain; charset=utf-8") + + return super().do_GET() + + +def main(): + port = int(sys.argv[1]) if len(sys.argv) > 1 else 8080 + ThreadingHTTPServer(("0.0.0.0", port), Handler).serve_forever() + + +if __name__ == "__main__": + main() diff --git a/service/test_server.py b/service/test_server.py new file mode 100644 index 0000000..e4e9f34 --- /dev/null +++ b/service/test_server.py @@ -0,0 +1,56 @@ +import http.client +import subprocess +import sys +import time +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +class UpdateServerTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.process = subprocess.Popen([sys.executable, "service/server.py", "18080"], cwd=ROOT) + for _ in range(50): + try: + conn = http.client.HTTPConnection("127.0.0.1", 18080, timeout=1) + conn.request("GET", "/healthz") + if conn.getresponse().status == 200: + conn.close() + return + except OSError: + pass + time.sleep(0.1) + raise RuntimeError("update server did not start") + + @classmethod + def tearDownClass(cls): + cls.process.terminate() + cls.process.wait(timeout=5) + + def get(self, path): + conn = http.client.HTTPConnection("127.0.0.1", 18080, timeout=5) + conn.request("GET", path) + response = conn.getresponse() + result = response.status, response.getheader("Content-Type"), response.read() + conn.close() + return result + + def test_latest_channel_wire_format(self): + status, content_type, body = self.get("/routeros/NEWESTa7.stable?version=7.23.8") + self.assertEqual(status, 200) + self.assertTrue(content_type.startswith("text/plain")) + self.assertEqual(body, b"7.25.1 1793623193\n") + + def test_old_series_is_available_for_changelog(self): + status, content_type, body = self.get("/routeros/7.23.8/CHANGELOG") + self.assertEqual(status, 200) + self.assertTrue(content_type.startswith("text/plain")) + self.assertTrue(body.startswith(b"What's new in 7.23.8")) + + def test_unknown_channel_does_not_claim_a_release(self): + status, _, body = self.get("/routeros/NEWESTa7.development") + self.assertEqual(status, 200) + self.assertIn(b"No version available", body) diff --git a/site/feed/changelogs/7.23.8.txt b/site/feed/changelogs/7.23.8.txt new file mode 100644 index 0000000..38953b8 --- /dev/null +++ b/site/feed/changelogs/7.23.8.txt @@ -0,0 +1,11 @@ +What's new in 7.23.8 (2026-10-08): + +*) console - fix console output of /system/identity/print being split into multiple lines (introduced in v7.23.6); +*) ethernet - improved stability when forcing unsupported link speed; +*) ipsec - fix IPsec tunnels not establishing when an address my-id is configured on the responder; +*) ospf - fix unset interface template parameters not being applied to interfaces; +*) quickset - fix the WAN interface being selected incorrectly; +*) system - improve stability (includes CVE-2026-84411, CVE-2026-103903); +*) system - stop the flash configuration store from growing on dynamic configuration changes; +*) wifi - fix malformed management frame information elements advertising availability of multiple 6GHz APs; +*) wifi - update radio regulatory information; \ No newline at end of file diff --git a/site/feed/changelogs/7.24.6.txt b/site/feed/changelogs/7.24.6.txt new file mode 100644 index 0000000..5af3c9b --- /dev/null +++ b/site/feed/changelogs/7.24.6.txt @@ -0,0 +1,22 @@ +What's new in 7.24.6 (2026-10-30): + +*) bridge - fix BPDUs leaking to the wrong dual-connected bond in MLAG (introduced in v7.24); +*) bridge - fix VLAN-tagged traffic being looped back in MLAG setups on CRS8xx series switches; +*) dude - fix the server not starting at boot when data-directory is on a USB disk; +*) ethernet - fix occasional packet loss on hAP be3 Media; +*) fetch - fix HTTP fetch getting stuck in a "bad HTTP response" state; +*) ipsec - fix IKEv2 EAP-TTLS client failing with invalid trust store (introduced in v7.21); +*) ipsec - fix IPsec tunnels failing after migration from the old QKD settings; +*) ipsec - fix IPsec tunnels not establishing when an address my-id is configured on the responder; +*) ptp - fix PTP support on CRS804-4DDQ; +*) quickset - fix the WAN interface being selected incorrectly; +*) route - improve static route handling performance; +*) routing-filter - fix any-regexp community matcher matching an empty community attribute; +*) routing-filter - fix community delete, filter and modify with regexp-only community lists; +*) routing-filter - fix host matching in address lists with more than 64 ranges; +*) sfp - fix DAC EEPROM corruption when disabling 400G ports on CRS804 and CRS812 switches; +*) system - improve stability (includes CVE-2026-98432); +*) system - stop the flash configuration store from growing on dynamic configuration changes; +*) wifi - reuse freed station association IDs; +*) wifi-mediatek - fix roaming between APs belonging to the same MLO group; +*) winbox - fix bridge filter window disabling valid fields for IPv6 rules; \ No newline at end of file diff --git a/site/feed/changelogs/7.25.1.txt b/site/feed/changelogs/7.25.1.txt new file mode 100644 index 0000000..6a9610a --- /dev/null +++ b/site/feed/changelogs/7.25.1.txt @@ -0,0 +1,18 @@ +What's new in 7.25.1 (2026-11-02): + +*) bridge - fix BPDUs leaking to the wrong dual-connected bond in MLAG (introduced in v7.24); +*) bridge - improve FDB synchronization between MLAG peers when VLANs are added; +*) console - fix custom SSH client login parameters being ignored (introduced in v7.25); +*) dhcpv4-client - fix packet length calculation (introduced in v7.25); +*) dude - fix missing outage times and graph background scaling (introduced in v7.25); +*) fetch - fix HTTP fetch getting stuck in a "bad HTTP response" state; +*) ipsec - fix XFRM interfaces staying down until a reboot; +*) ipv6 - fix from-pool address lifetimes not updating after a prefix renewal (introduced in v7.25); +*) pim - fix IGMP/MLD querier being disabled after upgrade (introduced in v7.25); +*) ptp - fix PTP support on CRS804-4DDQ; +*) routing-filter - fix host matching in address lists with more than 64 ranges; +*) sfp - fix DAC EEPROM corruption when disabling 400G ports on CRS804 and CRS812 switches; +*) ssh - fix packet alignment when SHA-1 is used (introduced in v7.25); +*) system - improve stability (includes CVE-2026-98432); +*) webfig - fix multi-value fields being displayed as empty (introduced in v7.25); +*) wifi - fix virtual interface restart loop when reselect-interval and reselect-time are configured (introduced in v7.25); \ No newline at end of file diff --git a/site/feed/channels.json b/site/feed/channels.json new file mode 100644 index 0000000..e2b087a --- /dev/null +++ b/site/feed/channels.json @@ -0,0 +1,5 @@ +{ + "stable": {"version": "7.25.1", "epoch": 1793623193}, + "long-term": {"version": "7.24.6", "epoch": 1793348669}, + "testing": {"version": "7.23.8", "epoch": 1791452776} +} diff --git a/site/releases.js b/site/releases.js index dccc709..c89266c 100644 --- a/site/releases.js +++ b/site/releases.js @@ -14,7 +14,7 @@ function safeLink(url) { } async function load() { try { - const response = await fetch('https://code.brunner.ninja/api/v1/repos/feedc0de/mikrotik-olsr/releases?limit=10', {signal: AbortSignal.timeout(12000)}); + const response = await fetch('/api/releases', {signal: AbortSignal.timeout(12000)}); if (!response.ok) throw new Error('Release service unavailable'); const releases = (await response.json()).filter(release => !release.draft); status.textContent = releases.length ? 'Read the release notes before choosing packages. Prereleases may have incomplete hardware validation.' : 'No releases have been published yet. Follow the implementation status or build from source; there are no prebuilt downloads to offer yet.'; diff --git a/test-site.sh b/test-site.sh new file mode 100755 index 0000000..54b5b13 --- /dev/null +++ b/test-site.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +helm lint chart/mikrotik-olsr-site +rendered=$(helm template mikrotik-olsr-site chart/mikrotik-olsr-site \ + --set ingress.enabled=true --set ingress.host=mikrotik.example.invalid) +printf '%s\n' "$rendered" | rg -q 'kind: Deployment' +printf '%s\n' "$rendered" | rg -q 'kind: Service' +printf '%s\n' "$rendered" | rg -q 'kind: Ingress' +python3 -m unittest service.test_server -v diff --git a/tools/build-bird.sh b/tools/build-bird.sh index 64e47a4..86b03af 100755 --- a/tools/build-bird.sh +++ b/tools/build-bird.sh @@ -2,6 +2,7 @@ # Local BGP bridge for native RouterOS dynamic route ownership. set -eu ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +BUILD_DIR=${BUILD_DIR:-$ROOT/build} CACHE="$ROOT/build/runtime-sources" : "${CROSS_COMPILE:=$CACHE/arm-linux-musleabi-cross/bin/arm-linux-musleabi-}" VERSION=2.20.0 @@ -13,16 +14,17 @@ fi printf '%s %s\n' 71c7ed4a04b7b8335b2e659246a32078d6c3f12249c88254e165c4033b353190 "$ARCHIVE" | sha256sum -c - if [ ! -d "$CACHE/bird-$VERSION" ]; then tar -xzf "$ARCHIVE" -C "$CACHE"; fi cd "$CACHE/bird-$VERSION" +TARGET=${TARGET:-arm-linux-musleabi} CC="${CROSS_COMPILE}gcc" AR="${CROSS_COMPILE}ar" RANLIB="${CROSS_COMPILE}ranlib" \ CFLAGS='-Os -ffunction-sections -fdata-sections' LDFLAGS='-static -Wl,--gc-sections' \ - ./configure --host=arm-linux-musleabi --disable-client --disable-libssh \ + ./configure --host="$TARGET" --disable-client --disable-libssh \ --disable-mpls-kernel --with-protocols=bgp,static make -j"${JOBS:-2}" make birdcl -mkdir -p "$ROOT/build/bird-root/bin" "$ROOT/build/bird-root/share/licenses" -"${CROSS_COMPILE}strip" -o "$ROOT/build/bird-root/bin/bird" bird -"${CROSS_COMPILE}strip" -o "$ROOT/build/bird-root/bin/birdcl" birdcl -cp README "$ROOT/build/bird-root/share/licenses/bird-README.txt" -cp "$ROOT/LICENSE" "$ROOT/build/bird-root/share/licenses/GPL-3.txt" +mkdir -p "$BUILD_DIR/bird-root/bin" "$BUILD_DIR/bird-root/share/licenses" +"${CROSS_COMPILE}strip" -o "$BUILD_DIR/bird-root/bin/bird" bird +"${CROSS_COMPILE}strip" -o "$BUILD_DIR/bird-root/bin/birdcl" birdcl +cp README "$BUILD_DIR/bird-root/share/licenses/bird-README.txt" +cp "$ROOT/LICENSE" "$BUILD_DIR/bird-root/share/licenses/GPL-3.txt" -cp -a "$ROOT/build/bird-root/." "$ROOT/build/olsr-root/" +cp -a "$BUILD_DIR/bird-root/." "$BUILD_DIR/olsr-root/" diff --git a/tools/build-olsrd.sh b/tools/build-olsrd.sh index 586e805..b46bfd8 100755 --- a/tools/build-olsrd.sh +++ b/tools/build-olsrd.sh @@ -3,6 +3,7 @@ set -eu : "${CROSS_COMPILE:?Set CROSS_COMPILE to an ARM Linux musl toolchain prefix}" ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +BUILD_DIR=${BUILD_DIR:-$ROOT/build} SOURCE="$ROOT/build/olsrd-source" REVISION=8d2a2500c2f626e45a7d343f2d8160784b34fb37 if [ ! -d "$SOURCE/.git" ]; then @@ -21,22 +22,21 @@ PY make -C "$SOURCE" clean make -C "$SOURCE" -j"${JOBS:-2}" OS=linux CC="${CROSS_COMPILE}gcc" \ STRIP="${CROSS_COMPILE}strip" LDFLAGS=-static DEBUG=0 olsrd -mkdir -p "$ROOT/build/olsr-root/bin" "$ROOT/build/olsr-root/share/licenses" -"${CROSS_COMPILE}strip" -o "$ROOT/build/olsr-root/bin/olsrd" "$SOURCE/olsrd" -cp "$SOURCE/license.txt" "$ROOT/build/olsr-root/share/licenses/olsrd.txt" -printf '%s\n' "$REVISION" > "$ROOT/build/olsr-root/share/olsrd-source-revision" +mkdir -p "$BUILD_DIR/olsr-root/bin" "$BUILD_DIR/olsr-root/share/licenses" +"${CROSS_COMPILE}strip" -o "$BUILD_DIR/olsr-root/bin/olsrd" "$SOURCE/olsrd" +cp "$SOURCE/license.txt" "$BUILD_DIR/olsr-root/share/licenses/olsrd.txt" +printf '%s\n' "$REVISION" > "$BUILD_DIR/olsr-root/share/olsrd-source-revision" printf '%s\n' 'Development payload: RouterOS startup and UI integration are not yet validated.' \ - > "$ROOT/build/olsr-root/share/STATUS.txt" + > "$BUILD_DIR/olsr-root/share/STATUS.txt" -cp -a "$ROOT/packages/olsr/bin/." "$ROOT/build/olsr-root/bin/" -mkdir -p "$ROOT/build/olsr-root/etc" -cp -a "$ROOT/packages/olsr/etc/." "$ROOT/build/olsr-root/etc/" +cp -a "$ROOT/packages/olsr/bin/." "$BUILD_DIR/olsr-root/bin/" +mkdir -p "$BUILD_DIR/olsr-root/etc" +cp -a "$ROOT/packages/olsr/etc/." "$BUILD_DIR/olsr-root/etc/" for helper in run-child log-ring start-service; do "${CROSS_COMPILE}gcc" -Os -static -Wall -Wextra -Werror \ - "$ROOT/packages/olsr/src/$helper.c" -o "$ROOT/build/olsr-root/bin/$helper" - "${CROSS_COMPILE}strip" "$ROOT/build/olsr-root/bin/$helper" + "$ROOT/packages/olsr/src/$helper.c" -o "$BUILD_DIR/olsr-root/bin/$helper" + "${CROSS_COMPILE}strip" "$BUILD_DIR/olsr-root/bin/$helper" done # Remove the retired standalone web control page from reused build directories. -rm -rf "$ROOT/build/olsr-root/home/web/olsr" -python3 "$ROOT/tools/build-webfig.py" --experimental --root "$ROOT/build/olsr-root" --version "${ROUTEROS_VERSION:-7.25.1}" +python3 "$ROOT/tools/build-webfig.py" --experimental --root "$BUILD_DIR/olsr-root" --version "${ROUTEROS_VERSION:-7.25.1}" diff --git a/tools/build-release.py b/tools/build-release.py index 4259cc7..6e10691 100755 --- a/tools/build-release.py +++ b/tools/build-release.py @@ -9,6 +9,8 @@ import shutil import subprocess import sys import tarfile +import time +import urllib.error import urllib.request import zipfile @@ -16,11 +18,35 @@ ROOT = Path(__file__).resolve().parents[1] def download(url, output): + def validate(path): + if path.name.endswith('.zip'): + with zipfile.ZipFile(path) as archive: + if archive.testzip() is not None: + raise zipfile.BadZipFile(f'CRC check failed for {path.name}') + elif path.name.endswith(('.tar.gz', '.tgz')): + with tarfile.open(path, 'r:gz') as archive: + archive.getmembers() + + temporary = output.with_suffix(output.suffix + '.part') + if output.exists(): + try: + validate(output) + except (OSError, tarfile.TarError, zipfile.BadZipFile): + output.unlink() if not output.exists(): - temporary = output.with_suffix(output.suffix + '.part') - with urllib.request.urlopen(url, timeout=60) as response, temporary.open('wb') as target: - shutil.copyfileobj(response, target) - temporary.replace(output) + temporary.unlink(missing_ok=True) + for attempt in range(3): + try: + with urllib.request.urlopen(url, timeout=120) as response, temporary.open('wb') as target: + shutil.copyfileobj(response, target) + validate(temporary) + temporary.replace(output) + break + except (OSError, urllib.error.URLError, tarfile.TarError, zipfile.BadZipFile): + temporary.unlink(missing_ok=True) + if attempt == 2: + raise + time.sleep(attempt + 1) return {'url': url, 'sha256': hashlib.sha256(output.read_bytes()).hexdigest()} @@ -33,6 +59,10 @@ def build(args): raise ValueError('Expected a RouterOS v7 version') version = args.version arch = args.arch + if args.wifi == 'wifi-qcom-ac' and arch != 'arm': + raise ValueError('wifi-qcom-ac is only selected by this project for ARMv7') + if args.wifi == 'wifi-qcom' and arch != 'arm64': + raise ValueError('wifi-qcom is only selected by this project for ARM64') cache = ROOT / 'build' / f'inputs-{version}-{arch}' output = Path(args.output).resolve() cache.mkdir(parents=True, exist_ok=True) @@ -111,7 +141,7 @@ def build(args): 'source_commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(), 'payloads': [name for name, directory in payloads.items() if directory]} (output / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n') - for name in ('RELEASE-NOTES.md', 'INSTALL.md'): + for name in ('RELEASE-NOTES.md', 'INSTALL.md', 'olsr.md', 'routeros-internals.md'): shutil.copyfile(ROOT / 'docs' / name, output / name) shutil.copyfile(ROOT / 'tools/configure-device.py', output / 'configure-device.py') checksums = ''.join(f'{hashlib.sha256(p.read_bytes()).hexdigest()} {p.name}\n' @@ -123,8 +153,9 @@ def build(args): def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--version', required=True) - p.add_argument('--arch', choices=['arm'], default='arm', help='Only ARM system packages have been exercised so far') - p.add_argument('--wifi', choices=['wifi-qcom-ac', 'wireless']) + p.add_argument('--arch', choices=['arm', 'arm64'], default='arm', + help='ARMv7 or ARM64/AArch64 system and add-on packages') + p.add_argument('--wifi', choices=['wifi-qcom-ac', 'wifi-qcom', 'wireless']) p.add_argument('--keys', required=True) p.add_argument('--output', required=True) p.add_argument('--option-root', help='Audited runtime payload directory') diff --git a/tools/build-runtime.sh b/tools/build-runtime.sh index 2366089..b4982c9 100755 --- a/tools/build-runtime.sh +++ b/tools/build-runtime.sh @@ -3,18 +3,53 @@ set -eu ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) CACHE="$ROOT/build/runtime-sources" +ARCH=${1:-${ROUTEROS_ARCH:-arm}} +BUILD_DIR="$ROOT/build/runtime-$ARCH" mkdir -p "$CACHE" # Always build fresh payload trees so removed files cannot leak into releases. -rm -rf "$ROOT/build/option-root" "$ROOT/build/olsr-root" "$ROOT/build/bird-root" +rm -rf "$BUILD_DIR/option-root" "$BUILD_DIR/olsr-root" "$BUILD_DIR/bird-root" +mkdir -p "$BUILD_DIR" fetch() { url=$1 file=$2 checksum=$3 - if [ ! -f "$file" ]; then curl --fail --location "$url" -o "$file"; fi + if [ -f "$file" ] && ! printf '%s %s\n' "$checksum" "$file" | sha256sum -c - >/dev/null 2>&1; then + rm -f "$file" + fi + # Reuse the original ARM toolchain cache name used by previous builds. + # This avoids downloading the same 100 MB archive again after a cache-key change. + if [ ! -f "$file" ] && [ -f "$CACHE/toolchain.tgz" ] && + printf '%s %s\n' "$checksum" "$CACHE/toolchain.tgz" | sha256sum -c - >/dev/null 2>&1; then + cp "$CACHE/toolchain.tgz" "$file" + fi + if [ ! -f "$file" ]; then + tmp="$file.part" + rm -f "$tmp" + curl --fail --location --retry 3 --retry-delay 2 "$url" -o "$tmp" + printf '%s %s\n' "$checksum" "$tmp" | sha256sum -c - + mv "$tmp" "$file" + fi printf '%s %s\n' "$checksum" "$file" | sha256sum -c - } -fetch https://musl.cc/arm-linux-musleabi-cross.tgz "$CACHE/toolchain.tgz" \ - d70c607101fee5330083463feacf5992892a85b826d1626094500e0c37ec7d25 -if [ ! -d "$CACHE/arm-linux-musleabi-cross" ]; then tar -xzf "$CACHE/toolchain.tgz" -C "$CACHE"; fi -export CROSS_COMPILE="$CACHE/arm-linux-musleabi-cross/bin/arm-linux-musleabi-" +case "$ARCH" in + arm) + TOOLCHAIN=arm-linux-musleabi-cross + TOOLCHAIN_URL=https://musl.cc/arm-linux-musleabi-cross.tgz + TOOLCHAIN_SHA=d70c607101fee5330083463feacf5992892a85b826d1626094500e0c37ec7d25 + TARGET=arm-linux-musleabi + ;; + arm64) + TOOLCHAIN=aarch64-linux-musl-cross + TOOLCHAIN_URL=https://musl.cc/aarch64-linux-musl-cross.tgz + TOOLCHAIN_SHA=c909817856d6ceda86aa510894fa3527eac7989f0ef6e87b5721c58737a06c38 + TARGET=aarch64-linux-musl + ;; + *) echo "Unsupported runtime architecture: $ARCH" >&2; exit 2 ;; +esac +fetch "$TOOLCHAIN_URL" "$CACHE/$TOOLCHAIN.tgz" "$TOOLCHAIN_SHA" +if [ ! -d "$CACHE/$TOOLCHAIN" ]; then tar -xzf "$CACHE/$TOOLCHAIN.tgz" -C "$CACHE"; fi +export CROSS_COMPILE="$CACHE/$TOOLCHAIN/bin/$TARGET-" +export ROUTEROS_ARCH="$ARCH" +export TARGET +export BUILD_DIR fetch https://busybox.net/downloads/busybox-1.37.0.tar.bz2 "$CACHE/busybox.tar.bz2" \ 3311dff32e746499f4df0d5df04d7eb396382d7e108bb9250e7b519b837043a4 if [ ! -d "$CACHE/busybox-1.37.0" ]; then tar -xjf "$CACHE/busybox.tar.bz2" -C "$CACHE"; fi @@ -44,25 +79,25 @@ subprocess.run(['make', '-C', sys.argv[1], 'CROSS_COMPILE=' + sys.argv[2], 'oldc input=b'\n' * 4096, check=True, stdout=subprocess.DEVNULL) PY make -C "$SOURCE" -j"${JOBS:-2}" CROSS_COMPILE="$CROSS_COMPILE" -mkdir -p "$ROOT/build/option-root/bin" "$ROOT/build/option-root/share/licenses" -cp "$SOURCE/busybox" "$ROOT/build/option-root/bin/busybox" -ln -sfn busybox "$ROOT/build/option-root/bin/bash" -ln -sfn busybox "$ROOT/build/option-root/bin/sh" +mkdir -p "$BUILD_DIR/option-root/bin" "$BUILD_DIR/option-root/share/licenses" +cp "$SOURCE/busybox" "$BUILD_DIR/option-root/bin/busybox" +ln -sfn busybox "$BUILD_DIR/option-root/bin/bash" +ln -sfn busybox "$BUILD_DIR/option-root/bin/sh" # Make enabled applets usable in the interactive shell too. for applet in cat chmod cp date echo grep head id ip kill ls mkdir mkfifo mount mv printf readlink rm sed sha256sum sleep stat tail test timeout umount uname; do - ln -sfn busybox "$ROOT/build/option-root/bin/$applet" + ln -sfn busybox "$BUILD_DIR/option-root/bin/$applet" done -cp "$SOURCE/LICENSE" "$ROOT/build/option-root/share/licenses/busybox.txt" -cp "$SOURCE/.config" "$ROOT/build/option-root/share/busybox.config" +cp "$SOURCE/LICENSE" "$BUILD_DIR/option-root/share/licenses/busybox.txt" +cp "$SOURCE/.config" "$BUILD_DIR/option-root/share/busybox.config" printf '%s\n' 'BusyBox utilities only. No license generator, shell unlock, or loader patch.' \ - > "$ROOT/build/option-root/share/STATUS.txt" + > "$BUILD_DIR/option-root/share/STATUS.txt" "$ROOT/tools/build-olsrd.sh" "$ROOT/tools/build-bird.sh" # Attribution for statically linked toolchain runtime libraries. for payload in option olsr; do - cp "$ROOT/vendor/licenses/musl-COPYRIGHT" "$ROOT/build/$payload-root/share/licenses/" - cp "$ROOT/vendor/licenses/GCC-RUNTIME-EXCEPTION" "$ROOT/build/$payload-root/share/licenses/" - cp "$ROOT/LICENSE" "$ROOT/build/$payload-root/share/licenses/GPL-3.txt" + cp "$ROOT/vendor/licenses/musl-COPYRIGHT" "$BUILD_DIR/$payload-root/share/licenses/" + cp "$ROOT/vendor/licenses/GCC-RUNTIME-EXCEPTION" "$BUILD_DIR/$payload-root/share/licenses/" + cp "$ROOT/LICENSE" "$BUILD_DIR/$payload-root/share/licenses/GPL-3.txt" done