diff --git a/.github/workflows/image-build.yml b/.github/workflows/image-build.yml index 6fef53d..dc38f09 100644 --- a/.github/workflows/image-build.yml +++ b/.github/workflows/image-build.yml @@ -35,7 +35,6 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ secrets.AWS_REGION }} S3_BUCKET: ${{ secrets.S3_BUCKET }} - S3_PREFIX: ${{ secrets.S3_PREFIX }} run: | key="$(scripts/upload-image.sh)" echo "S3_KEY=${key}" >> "${GITHUB_ENV}" diff --git a/AGENTS.md b/AGENTS.md index 2db86aa..d79a942 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,7 +44,7 @@ Deploy flow (automation-first): - Bootstrap S3 bucket + scoped IAM user + VM Import role with `infra/opentofu/aws` (use homelab-admin creds). - Import the image into AWS as an AMI (`aws ec2 import-image`). - Grab the host SSH key and add it to `../nix/nix-secrets/secrets.nix`; rekey secrets with agenix. -- Ensure required secrets exist: `clawdinator-github-app.pem`, `clawdinator-discord-token`, `anthropic-api-key`. +- Ensure required secrets exist: `clawdinator-github-app.pem`, `clawdinator-discord-token`, `clawdinator-anthropic-api-key`. - Update `nix/hosts/.nix` (Discord allowlist, GitHub App installationId, identity name). - Ensure `/var/lib/clawd/repo` contains this repo (self-update requires it). - Verify systemd services: `clawdinator`, `clawdinator-github-app-token`, `clawdinator-self-update`. @@ -53,4 +53,4 @@ Deploy flow (automation-first): Key principle: mental notes don’t survive restarts — write it to a file. Cattle vs pets: hosts are disposable. Prefer re-provisioning from OpenTofu + NixOS configs over in-place manual fixes. -One way only: AWS AMI pipeline via S3 + VM Import. This is a greenfield repo. Do not reference "existing", "legacy", or alternate paths anywhere in code or docs. +One way only: AWS AMI pipeline via S3 + VM Import. This is a greenfield repo. Do not reference alternate paths anywhere in code or docs. diff --git a/README.md b/README.md index f26ecfd..6d91a61 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Deploy (automation‑first): - Ensure `/var/lib/clawd/repo` contains this repo (needed for self‑update). - Configure Discord guild/channel allowlist and GitHub App installation ID. -Image-based deploy (Option A, recommended): +Image-based deploy (only path): 1) Build a bootstrap image with nixos-generators: - `nix run github:nix-community/nixos-generators -- -f amazon -c nix/hosts/clawdinator-1-image.nix -o dist` 2) Upload the raw image to S3 (private object). diff --git a/docs/SECRETS.md b/docs/SECRETS.md index e73c5e9..7a2d89e 100644 --- a/docs/SECRETS.md +++ b/docs/SECRETS.md @@ -8,7 +8,6 @@ Infrastructure (OpenTofu): Image pipeline (CI): - `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` / `AWS_REGION` / `S3_BUCKET` (required). -- `S3_PREFIX` (optional). Local storage: - Keep AWS keys encrypted in `../nix/nix-secrets` for local runs if needed. @@ -33,8 +32,8 @@ Agenix (local secrets repo): - Store encrypted files in `../nix/nix-secrets` (relative to this repo). - Sync encrypted secrets to the host at `/var/lib/clawd/nix-secrets`. - Decrypt on host with agenix; point NixOS options at `/run/agenix/*`. -- Required files (minimum): `clawdinator-github-app.pem.age`, `clawdinator-discord-token.age`, `clawdis-anthropic-api-key.age`. -- CI image pipeline (stored locally, not on hosts): `clawdinator-ami-importer-access-key-id.age`, `clawdinator-ami-importer-secret-access-key.age`, `clawdinator-image-bucket-name.age`, `clawdinator-image-bucket-region.age`. +- Required files (minimum): `clawdinator-github-app.pem.age`, `clawdinator-discord-token.age`, `clawdinator-anthropic-api-key.age`. +- CI image pipeline (stored locally, not on hosts): `clawdinator-image-uploader-access-key-id.age`, `clawdinator-image-uploader-secret-access-key.age`, `clawdinator-image-bucket-name.age`, `clawdinator-image-bucket-region.age`. Example NixOS wiring (agenix): ``` @@ -44,15 +43,15 @@ Example NixOS wiring (agenix): age.secrets."clawdinator-github-app.pem".file = "/var/lib/clawd/nix-secrets/clawdinator-github-app.pem.age"; - age.secrets."clawdis-anthropic-api-key".file = - "/var/lib/clawd/nix-secrets/clawdis-anthropic-api-key.age"; + age.secrets."clawdinator-anthropic-api-key".file = + "/var/lib/clawd/nix-secrets/clawdinator-anthropic-api-key.age"; age.secrets."clawdinator-discord-token".file = "/var/lib/clawd/nix-secrets/clawdinator-discord-token.age"; services.clawdinator.githubApp.privateKeyFile = "/run/agenix/clawdinator-github-app.pem"; services.clawdinator.anthropicApiKeyFile = - "/run/agenix/clawdis-anthropic-api-key"; + "/run/agenix/clawdinator-anthropic-api-key"; services.clawdinator.discordTokenFile = "/run/agenix/clawdinator-discord-token"; } diff --git a/docs/SHARED_MEMORY.md b/docs/SHARED_MEMORY.md index 015d2ef..8496246 100644 --- a/docs/SHARED_MEMORY.md +++ b/docs/SHARED_MEMORY.md @@ -7,7 +7,7 @@ POC recommendation: - Memory lives at /var/lib/clawd/memory. File patterns: -- Daily notes (optionally per instance): YYYY-MM-DD_INSTANCE.md +- Daily notes can be per instance: YYYY-MM-DD_INSTANCE.md (merge later). - Canonical knowledge (single shared files): - project.md (goals + non-negotiables) - architecture.md @@ -23,11 +23,11 @@ Example layout: │ ├── architecture.md │ ├── discord.md │ ├── whatsapp.md -│ └── 2026-01-06_CLAWDINATOR-1.md +│ └── 2026-01-06.md ``` AGENTS.md should reference key memory files explicitly (e.g., “For Discord context, also read memory/discord.md”). -Later scale options: -- Shared filesystem or object storage sync with file locking. +Multi-host requirement: +- Use a shared filesystem or object storage sync with file locking. - Keep canonical files authoritative; merge per-instance notes periodically. diff --git a/flake.nix b/flake.nix index cda4cf8..c3c8ec4 100644 --- a/flake.nix +++ b/flake.nix @@ -5,14 +5,13 @@ nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; nix-clawdbot.url = "github:clawdbot/nix-clawdbot"; # latest upstream agenix.url = "github:ryantm/agenix"; - disko.url = "github:nix-community/disko"; secrets = { url = "path:../nix/nix-secrets"; flake = false; }; }; - outputs = { self, nixpkgs, nix-clawdbot, agenix, disko, secrets }: + outputs = { self, nixpkgs, nix-clawdbot, agenix, secrets }: let lib = nixpkgs.lib; systems = [ "x86_64-linux" "aarch64-linux" ]; @@ -88,17 +87,8 @@ modules = [ ({ ... }: { nixpkgs.overlays = [ self.overlays.default ]; }) agenix.nixosModules.default - disko.nixosModules.disko ./nix/hosts/clawdinator-1.nix ]; }; - - nixosConfigurations.clawdinator-1-bootstrap = nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = [ - disko.nixosModules.disko - ./nix/hosts/clawdinator-1-bootstrap.nix - ]; - }; }; } diff --git a/nix/examples/clawdinator-host.nix b/nix/examples/clawdinator-host.nix index 74cca1f..e9e4b9e 100644 --- a/nix/examples/clawdinator-host.nix +++ b/nix/examples/clawdinator-host.nix @@ -2,8 +2,8 @@ { age.secrets."clawdinator-github-app.pem".file = "/var/lib/clawd/nix-secrets/clawdinator-github-app.pem.age"; - age.secrets."clawdis-anthropic-api-key".file = - "/var/lib/clawd/nix-secrets/clawdis-anthropic-api-key.age"; + age.secrets."clawdinator-anthropic-api-key".file = + "/var/lib/clawd/nix-secrets/clawdinator-anthropic-api-key.age"; age.secrets."clawdinator-discord-token".file = "/var/lib/clawd/nix-secrets/clawdinator-discord-token.age"; @@ -40,7 +40,7 @@ }; }; - anthropicApiKeyFile = "/run/agenix/clawdis-anthropic-api-key"; + anthropicApiKeyFile = "/run/agenix/clawdinator-anthropic-api-key"; discordTokenFile = "/run/agenix/clawdinator-discord-token"; githubApp = { diff --git a/nix/hosts/clawdinator-1-bootstrap.nix b/nix/hosts/clawdinator-1-bootstrap.nix deleted file mode 100644 index 59a314e..0000000 --- a/nix/hosts/clawdinator-1-bootstrap.nix +++ /dev/null @@ -1,52 +0,0 @@ -{ config, ... }: -{ - networking.hostName = "clawdinator-1"; - time.timeZone = "UTC"; - system.stateVersion = "26.05"; - - boot.loader.systemd-boot.enable = true; - boot.loader.grub.enable = false; - boot.loader.efi.canTouchEfiVariables = false; - boot.loader.efi.efiSysMountPoint = "/boot"; - - disko.devices = { - disk.main = { - type = "disk"; - device = "/dev/disk/by-path/pci-0000:06:00.0-scsi-0:0:0:0"; - content = { - type = "gpt"; - partitions = { - boot = { - size = "512M"; - type = "EF00"; - content = { - type = "filesystem"; - format = "vfat"; - mountpoint = "/boot"; - }; - }; - root = { - size = "100%"; - content = { - type = "filesystem"; - format = "ext4"; - mountpoint = "/"; - }; - }; - }; - }; - }; - }; - - boot.kernelParams = [ "net.ifnames=0" "biosdevname=0" ]; - networking.useDHCP = false; - networking.useNetworkd = false; - systemd.network.enable = false; - networking.interfaces.eth0.useDHCP = true; - - services.openssh.enable = true; - networking.firewall.enable = false; - users.users.root.openssh.authorizedKeys.keys = [ - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOLItFT3SVm5r7gELrfRRJxh6V2sf/BIx7HKXt6oVWpB" - ]; -} diff --git a/nix/hosts/clawdinator-1-image.nix b/nix/hosts/clawdinator-1-image.nix index 2fccdbf..430e520 100644 --- a/nix/hosts/clawdinator-1-image.nix +++ b/nix/hosts/clawdinator-1-image.nix @@ -1,28 +1,10 @@ { modulesPath, ... }: { imports = [ - (modulesPath + "/profiles/qemu-guest.nix") + (modulesPath + "/virtualisation/amazon-image.nix") ]; networking.hostName = "clawdinator-1"; time.timeZone = "UTC"; system.stateVersion = "26.05"; - - boot.loader.systemd-boot.enable = true; - boot.loader.grub.enable = false; - boot.loader.efi.canTouchEfiVariables = false; - boot.loader.efi.efiSysMountPoint = "/boot"; - - networking.useDHCP = false; - systemd.network.enable = true; - systemd.network.networks."10-ethernet" = { - matchConfig.Name = [ "en*" "eth*" ]; - networkConfig.DHCP = "yes"; - }; - services.openssh.enable = true; - networking.firewall.enable = false; - - users.users.root.openssh.authorizedKeys.keys = [ - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOLItFT3SVm5r7gELrfRRJxh6V2sf/BIx7HKXt6oVWpB" - ]; } diff --git a/nix/hosts/clawdinator-1.nix b/nix/hosts/clawdinator-1.nix index 7e0564f..80c51fb 100644 --- a/nix/hosts/clawdinator-1.nix +++ b/nix/hosts/clawdinator-1.nix @@ -1,41 +1,18 @@ -{ config, lib, pkgs, secrets, ... }: +{ modulesPath, pkgs, ... }: { - imports = [ ../modules/clawdinator.nix ]; + imports = [ + (modulesPath + "/virtualisation/amazon-image.nix") + ../modules/clawdinator.nix + ]; networking.hostName = "clawdinator-1"; - networking.useDHCP = false; - networking.useNetworkd = true; - systemd.network.enable = true; - systemd.network.networks."10-wan" = { - matchConfig.Type = "ether"; - networkConfig.DHCP = "yes"; - }; time.timeZone = "UTC"; system.stateVersion = "26.05"; - boot.loader.systemd-boot.enable = true; - boot.loader.grub.enable = false; - boot.loader.efi.canTouchEfiVariables = false; - boot.loader.efi.efiSysMountPoint = "/boot"; - - fileSystems."/" = { - device = "/dev/disk/by-partlabel/disk-main-root"; - fsType = "ext4"; - }; - - fileSystems."/boot" = { - device = "/dev/disk/by-partlabel/disk-main-boot"; - fsType = "vfat"; - }; - nix.package = pkgs.nixVersions.stable; nix.settings.experimental-features = [ "nix-command" "flakes" ]; - services.openssh.enable = true; networking.firewall.allowedTCPPorts = [ 22 18789 ]; - users.users.root.openssh.authorizedKeys.keys = [ - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOLItFT3SVm5r7gELrfRRJxh6V2sf/BIx7HKXt6oVWpB" - ]; age.identityPaths = [ "/etc/ssh/ssh_host_ed25519_key" ]; age.secrets."clawdinator-github-app.pem" = { @@ -43,8 +20,8 @@ owner = "clawdinator"; group = "clawdinator"; }; - age.secrets."clawdis-anthropic-api-key" = { - file = "/var/lib/clawd/nix-secrets/clawdis-anthropic-api-key.age"; + age.secrets."clawdinator-anthropic-api-key" = { + file = "/var/lib/clawd/nix-secrets/clawdinator-anthropic-api-key.age"; owner = "clawdinator"; group = "clawdinator"; }; @@ -88,7 +65,7 @@ }; }; - anthropicApiKeyFile = "/run/agenix/clawdis-anthropic-api-key"; + anthropicApiKeyFile = "/run/agenix/clawdinator-anthropic-api-key"; discordTokenFile = "/run/agenix/clawdinator-discord-token"; githubApp = { diff --git a/scripts/build-image.sh b/scripts/build-image.sh index d9b3b0e..3816a6d 100755 --- a/scripts/build-image.sh +++ b/scripts/build-image.sh @@ -11,10 +11,31 @@ fi nix run github:nix-community/nixos-generators -- -f "${format}" -c "${config_path}" -o "${out_dir}" -image_file="$(find "${out_dir}" -maxdepth 2 -type f \( -name "*.img" -o -name "*.vhd" -o -name "*.vhdx" -o -name "*.raw" \) | head -n 1)" +image_file="$(find "${out_dir}" -maxdepth 2 -type f \( -name "*.img" -o -name "*.vhd" -o -name "*.raw" -o -name "*.vmdk" \) | head -n 1)" if [ -z "${image_file}" ]; then echo "No image found in ${out_dir} for format ${format}" >&2 exit 1 fi -cp -f "${image_file}" "${out_dir}/nixos.img" +ext="${image_file##*.}" +ext="$(printf '%s' "${ext}" | tr '[:upper:]' '[:lower:]')" +case "${ext}" in + img|raw) + aws_format="raw" + ;; + vhd) + aws_format="vhd" + ;; + vmdk) + aws_format="vmdk" + ;; + *) + echo "Unsupported image extension: ${ext}" >&2 + exit 1 + ;; +esac + +image_target="${out_dir}/nixos.${ext}" +cp -f "${image_file}" "${image_target}" +printf '%s' "${image_target}" > "${out_dir}/image-path" +printf '%s' "${aws_format}" > "${out_dir}/image-format" diff --git a/scripts/import-image.sh b/scripts/import-image.sh index fa5e815..8f46d29 100755 --- a/scripts/import-image.sh +++ b/scripts/import-image.sh @@ -7,6 +7,26 @@ region="${AWS_REGION:?AWS_REGION required}" boot_mode="legacy-bios" arch="${AMI_ARCH:-x86_64}" +format="${IMAGE_FORMAT:-}" +if [ -z "${format}" ]; then + ext="${key##*.}" + ext="$(printf '%s' "${ext}" | tr '[:upper:]' '[:lower:]')" + case "${ext}" in + img|raw) + format="raw" + ;; + vhd) + format="vhd" + ;; + vmdk) + format="vmdk" + ;; + *) + echo "Unable to infer image format from S3 key: ${key}" >&2 + exit 1 + ;; + esac +fi timestamp="$(date -u +%Y%m%d%H%M%S)" ami_name="${AMI_NAME:-clawdinator-nixos-${timestamp}}" @@ -19,7 +39,7 @@ task_id="$( --boot-mode "${boot_mode}" \ --architecture "${arch}" \ --role-name "vmimport" \ - --disk-containers "Format=raw,UserBucket={S3Bucket=${bucket},S3Key=${key}}" \ + --disk-containers "Format=${format},UserBucket={S3Bucket=${bucket},S3Key=${key}}" \ --query 'ImportTaskId' \ --output text )" diff --git a/scripts/upload-image.sh b/scripts/upload-image.sh index 9eb13d9..4b98898 100755 --- a/scripts/upload-image.sh +++ b/scripts/upload-image.sh @@ -3,6 +3,9 @@ set -euo pipefail out_dir="${OUT_DIR:-dist}" image_path="${out_dir}/nixos.img" +if [ -f "${out_dir}/image-path" ]; then + image_path="$(cat "${out_dir}/image-path")" +fi if [ ! -f "${image_path}" ]; then echo "Expected image at ${image_path}" >&2 @@ -11,14 +14,11 @@ fi bucket="${S3_BUCKET:?S3_BUCKET required}" region="${AWS_REGION:?AWS_REGION required}" -prefix="${S3_PREFIX:-}" timestamp="$(date -u +%Y%m%d%H%M%S)" -key_prefix="${prefix%/}" -if [ -n "${key_prefix}" ]; then - key_prefix="${key_prefix}/" -fi -key="${key_prefix}clawdinator-nixos-${timestamp}.img" +ext="${image_path##*.}" +ext="$(printf '%s' "${ext}" | tr '[:upper:]' '[:lower:]')" +key="clawdinator-nixos-${timestamp}.${ext}" aws s3 cp "${image_path}" "s3://${bucket}/${key}" \ --region "${region}" \