Skip to content

Commit

Permalink
Merge pull request #49 from pedorich-n/fix-kmods
Browse files Browse the repository at this point in the history
Add support for separate Kmods
  • Loading branch information
astro authored Dec 13, 2024
2 parents d21509a + 46ab913 commit 65b3de7
Show file tree
Hide file tree
Showing 3 changed files with 132 additions and 71 deletions.
11 changes: 11 additions & 0 deletions builder.nix
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
(
import ./hashes/${release}.nix
).packages.${packagesArch}
# Attrset where key is kmodsTarget and value is checksum of `Packages` file. Required for OpenWRT >=24
, kmodsSha256 ? {}
# Extra OpenWRT packages (can be prefixed with "-")
, packages ? []
# Include extra files
Expand All @@ -36,8 +38,16 @@

let
inherit (pkgs) lib;

maybeKmodsSha256 = lib.optionalAttrs (lib.versionAtLeast release "24") (
if kmodsSha256 != { }
then kmodsSha256
else (import ./hashes/${release}.nix).kmods.${target}.${variant} or (builtins.throw "Failed to load Kmods hashes for ${target}/${variant}!")
);

inherit (import ./files.nix {
inherit pkgs release target variant sha256 feedsSha256 packagesArch;
kmodsSha256 = maybeKmodsSha256;
}) variantFiles profiles expandDeps corePackages packagesByFeed allPackages;

requiredPackages = profiles.default_packages or (
Expand All @@ -57,6 +67,7 @@ let
"sha256"
"packagesArch"
"feedsSha256"
"kmodsSha256"
"packages"
"files"
"disabledServices"
Expand Down
49 changes: 35 additions & 14 deletions files.nix
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
, sha256
# Checksum of a feed's `Packages` file
, feedsSha256
# Attrset where key is kmodsTarget and value is checksum of `Packages` file. Required for OpenWRT >=24
, kmodsSha256 ? {}
# Manually specify packages' arch for OpenWRT<19 releases without profiles.json
, packagesArch ? null
}:
Expand Down Expand Up @@ -87,25 +89,52 @@ let

baseUrl = "https://downloads.openwrt.org";
releaseUrl = if release == "snapshot" then "${baseUrl}/snapshots" else "${baseUrl}/releases/${release}";
variantFiles = fetchSums "${releaseUrl}/targets/${target}/${variant}" sha256;
targetVariantUrl = "${releaseUrl}/targets/${target}/${variant}";

variantFiles = fetchSums targetVariantUrl sha256;

profiles =
if variantFiles ? "profiles.json"
then lib.importJSON variantFiles."profiles.json"
else null;

arch =
if packagesArch == null
then profiles.arch_packages
else packagesArch;

kernelInfo = profiles.linux_kernel or (builtins.throw "No Kernel info found in profiles.json!");

kmodsTarget = "${kernelInfo.version}-${kernelInfo.release}-${kernelInfo.vermagic}";

kmodsVirtualFeed = lib.optionalAttrs (release == "snapshot" || lib.versionAtLeast release "24") {
"kmods" = kmodsSha256.${kmodsTarget} or (builtins.throw "Failed to resolve Kmods for ${kmodsTarget}");
};

allFeeds = feedsSha256 // kmodsVirtualFeed;

feedUrl = feed:
if (feed == "kmods")
then "${targetVariantUrl}/kmods/${kmodsTarget}"
else "${releaseUrl}/packages/${arch}/${feed}";

feedsPackagesFile = builtins.mapAttrs (feed: { sha256 }:
fetchurl {
url = "${releaseUrl}/packages/${arch}/${feed}/Packages";
url = "${feedUrl feed}/Packages";
inherit sha256;
}
) feedsSha256;
) allFeeds;

packagesByFeed = builtins.mapAttrs (feed: packagesFile:
parsePackages "${releaseUrl}/packages/${arch}/${feed}" (builtins.readFile packagesFile)
parsePackages (feedUrl feed) (builtins.readFile packagesFile)
) feedsPackagesFile;

corePackages =
parsePackages
"${releaseUrl}/targets/${target}/${variant}/packages"
"${targetVariantUrl}/packages"
(builtins.readFile variantFiles."packages/Packages");

realPackages =
realPackages =
(builtins.foldl' (a: b: a // b) { } (builtins.attrValues packagesByFeed))
// corePackages;

Expand Down Expand Up @@ -190,14 +219,6 @@ let
in
deps: builtins.attrNames (builtins.foldl' addDep { } (applyMinusDeps deps));

profiles =
if variantFiles ? "profiles.json"
then lib.importJSON variantFiles."profiles.json"
else null;

arch = if packagesArch == null
then profiles.arch_packages
else packagesArch;

in {
inherit allPackages corePackages packagesByFeed expandDeps variantFiles profiles arch;
Expand Down
143 changes: 86 additions & 57 deletions generate-hashes.nix
Original file line number Diff line number Diff line change
@@ -1,60 +1,89 @@
{ pkgs ? import <nixpkgs> {} }:

pkgs.writeShellScriptBin "generate-hashes" ''
PATH=${pkgs.lib.makeBinPath (with pkgs; [ jq curl nix ])}:$PATH
RELEASE="${import ./latest-release.nix}"
FEEDS="base luci packages routing telephony"
if [ $# -gt 0 ]; then
RELEASE=$1
fi
UPSTREAM_URL=https://downloads.openwrt.org
RELEASE_URL=$UPSTREAM_URL/releases/$RELEASE
if [ $RELEASE == "snapshot" ]; then
RELEASE_URL=$UPSTREAM_URL/snapshots;
fi
declare -A arches_fetched
hash() {
TARGET=$1
VARIANT=$2
echo "- $TARGET/$VARIANT" >&2
BASEURL=$RELEASE_URL/targets/$TARGET/$VARIANT
SUM=$(nix store prefetch-file --json $BASEURL/sha256sums 2>/dev/null | jq -r .hash)
if [ -n "$SUM" ]; then
echo " targets.\"$TARGET\".\"$VARIANT\".sha256 = \"$SUM\";"
ARCH=$(curl -s $BASEURL/profiles.json | jq -r .arch_packages)
[ $? -ne 0 ] && echo "failed to fetch or parse $BASEURL/profiles.json" > /dev/stderr
if [ -n "$ARCH" ]; then
echo " targets.\"$TARGET\".\"$VARIANT\".packagesArch = \"$ARCH\";"
if [ -z "''${arches_fetched[$ARCH]}" ]; then
for FEED in $FEEDS; do
echo " - $FEED" >&2
SUM=$(nix store prefetch-file --json $RELEASE_URL/packages/$ARCH/$FEED/Packages 2>/dev/null | jq -r .hash)
echo " packages.\"$ARCH\".\"$FEED\".sha256 = \"$SUM\";"
done
arches_fetched[$ARCH]="done"
{ pkgs ? import <nixpkgs> { } }:
pkgs.writeShellApplication {
name = "generate-hashes";
runtimeInputs = [
pkgs.curl
pkgs.jq
pkgs.nix
];

text = ''
RELEASE="''${1:-${import ./latest-release.nix}}"
FEEDS="base luci packages routing telephony"
UPSTREAM_URL=https://downloads.openwrt.org
RELEASE_URL="''${UPSTREAM_URL}/releases/''${RELEASE}"
if [ "''${RELEASE}" == "snapshot" ]; then
RELEASE_URL="''${UPSTREAM_URL}/snapshots";
fi
KMODS_SEPARATE=false
if [[ "''${RELEASE}" == "snapshot" || "''${RELEASE}" == 24* ]]; then
KMODS_SEPARATE=true
fi
declare -g -A arches_fetched
hash() {
TARGET=$1
VARIANT=$2
echo "- ''${TARGET}/''${VARIANT}" >&2
BASE_URL="''${RELEASE_URL}/targets/''${TARGET}/''${VARIANT}"
SUMS_HASH=$(nix store prefetch-file --json "''${BASE_URL}/sha256sums" 2>/dev/null | jq -r .hash)
if [ -n "$SUMS_HASH" ]; then
echo " targets.\"''${TARGET}\".\"''${VARIANT}\".sha256 = \"''${SUMS_HASH}\";"
PROFILES=$(curl --silent --fail "''${BASE_URL}/profiles.json" || true)
if [ -z "$PROFILES" ]; then
echo "Failed to fetch ''${BASE_URL}/profiles.json" >&2
else
if ''${KMODS_SEPARATE}; then
KERNEL_RELEASE=$(jq -r '.linux_kernel.release' <<< "''${PROFILES}")
KERNEL_VERSION=$(jq -r '.linux_kernel.version' <<< "''${PROFILES}")
KERNEL_VERMAGIC=$(jq -r '.linux_kernel.vermagic' <<< "''${PROFILES}")
KMODS_TARGET="''${KERNEL_VERSION}-''${KERNEL_RELEASE}-''${KERNEL_VERMAGIC}"
echo " - kmods" >&2
KMODS_HASH=$(nix store prefetch-file --json "''${BASE_URL}/kmods/''${KMODS_TARGET}/Packages" 2>/dev/null | jq -r .hash)
echo " kmods.\"''${TARGET}\".\"''${VARIANT}\".\"''${KMODS_TARGET}\".sha256 = \"''${KMODS_HASH}\";"
fi
ARCH=$(jq -r '.arch_packages' <<< "''${PROFILES}")
if [ -n "$ARCH" ]; then
echo " targets.\"''${TARGET}\".\"''${VARIANT}\".packagesArch = \"''${ARCH}\";"
if [ -z "''${arches_fetched[$ARCH]:-}" ]; then
for FEED in ''${FEEDS}; do
echo " - ''${FEED}" >&2
PACKAGES_HASH=$(nix store prefetch-file --json "''${RELEASE_URL}/packages/''${ARCH}/''${FEED}/Packages" 2>/dev/null | jq -r .hash)
echo " packages.\"''${ARCH}\".\"''${FEED}\".sha256 = \"''${PACKAGES_HASH}\";"
done
arches_fetched[$ARCH]="done"
fi
fi
fi
fi
}
mkdir -p hashes
(
echo "{"
curl -s "''${RELEASE_URL}/targets/?json-targets" | jq -r .[] | while IFS=/ read -r TARGET VARIANT; do
hash "''${TARGET}" "''${VARIANT}"
done
echo "}"
) > "hashes/''${RELEASE}.nix"
if [ "$(stat -c %s "hashes/''${RELEASE}.nix")" -le 10 ] ; then
# Too small, no entries, discard.
rm -v "hashes/''${RELEASE}.nix"
fi
fi
'';
}
mkdir -p hashes
(
echo "{"
curl -s $RELEASE_URL/targets/?json-targets | jq -r .[] | while IFS=/ read TARGET VARIANT; do
hash $TARGET $VARIANT
done
echo "}"
) > hashes/$RELEASE.nix
if [ $(stat -c %s hashes/$RELEASE.nix) -le 10 ] ; then
# Too small, no entries, discard.
rm -v hashes/$RELEASE.nix
fi
''

0 comments on commit 65b3de7

Please sign in to comment.