Public PR intent S3 bucket + publisher timer
- Provision public S3 bucket (anonymous list/get) for PR intent artifacts - Grant instance role PutObject and add NixOS systemd timer to publish /memory/pr-intent - Default agent thinking level to high for GPT-5.2/Codex - Make OpenTofu instance management explicit (manage_instances) to prevent accidental fleet destroy Tests: not run (infra/Nix changes)
This commit is contained in:
joshp123
parent
63fa64a0b1
commit
ffb27ab614
3
.gitignore
vendored
3
.gitignore
vendored
@ -1,3 +1,6 @@
|
||||
# Devenv
|
||||
.devenv*
|
||||
|
||||
# OpenTofu
|
||||
infra/opentofu/.terraform/
|
||||
infra/opentofu/.tofu/
|
||||
|
||||
123
devenv.lock
Normal file
123
devenv.lock
Normal file
@ -0,0 +1,123 @@
|
||||
{
|
||||
"nodes": {
|
||||
"devenv": {
|
||||
"locked": {
|
||||
"dir": "src/modules",
|
||||
"lastModified": 1771157881,
|
||||
"owner": "cachix",
|
||||
"repo": "devenv",
|
||||
"rev": "b0b3dfa70ec90fa49f672e579f186faf4f61bd4b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"dir": "src/modules",
|
||||
"owner": "cachix",
|
||||
"repo": "devenv",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-compat": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1767039857,
|
||||
"owner": "NixOS",
|
||||
"repo": "flake-compat",
|
||||
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"repo": "flake-compat",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"git-hooks": {
|
||||
"inputs": {
|
||||
"flake-compat": "flake-compat",
|
||||
"gitignore": "gitignore",
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1770726378,
|
||||
"owner": "cachix",
|
||||
"repo": "git-hooks.nix",
|
||||
"rev": "5eaaedde414f6eb1aea8b8525c466dc37bba95ae",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "cachix",
|
||||
"repo": "git-hooks.nix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"gitignore": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"git-hooks",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1762808025,
|
||||
"owner": "hercules-ci",
|
||||
"repo": "gitignore.nix",
|
||||
"rev": "cb5e3fdca1de58ccbc3ef53de65bd372b48f567c",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "hercules-ci",
|
||||
"repo": "gitignore.nix",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"inputs": {
|
||||
"nixpkgs-src": "nixpkgs-src"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1770434727,
|
||||
"owner": "cachix",
|
||||
"repo": "devenv-nixpkgs",
|
||||
"rev": "8430f16a39c27bdeef236f1eeb56f0b51b33d348",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "cachix",
|
||||
"ref": "rolling",
|
||||
"repo": "devenv-nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs-src": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1769922788,
|
||||
"narHash": "sha256-H3AfG4ObMDTkTJYkd8cz1/RbY9LatN5Mk4UF48VuSXc=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "207d15f1a6603226e1e223dc79ac29c7846da32e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixpkgs-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"devenv": "devenv",
|
||||
"git-hooks": "git-hooks",
|
||||
"nixpkgs": "nixpkgs",
|
||||
"pre-commit-hooks": [
|
||||
"git-hooks"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
||||
49
docs/PUBLIC_S3_PR_INTENT.md
Normal file
49
docs/PUBLIC_S3_PR_INTENT.md
Normal file
@ -0,0 +1,49 @@
|
||||
# Public S3: PR intent artifacts
|
||||
|
||||
Goal: publish a **public-safe** subtree from shared memory (`/memory`) to an **anonymous public S3 bucket** so maintainers can quickly list + pull artifacts.
|
||||
|
||||
## Infra (OpenTofu)
|
||||
|
||||
The AWS OpenTofu stack (`infra/opentofu/aws`) provisions a public bucket:
|
||||
- anonymous `ListBucket`
|
||||
- anonymous `GetObject`
|
||||
- CLAWDINATOR instance role can `PutObject` (no deletes)
|
||||
|
||||
After `tofu apply`, get the bucket name:
|
||||
|
||||
```sh
|
||||
tofu output -raw pr_intent_bucket_name
|
||||
```
|
||||
|
||||
## On-host publishing (NixOS)
|
||||
|
||||
Enable the publisher timer on CLAWDINATOR hosts:
|
||||
|
||||
```nix
|
||||
services.clawdinator.publicS3 = {
|
||||
enable = true;
|
||||
bucket = "<tofu output pr_intent_bucket_name>";
|
||||
# default sourceDir: "${config.services.clawdinator.memoryDir}/pr-intent"
|
||||
};
|
||||
```
|
||||
|
||||
Publishing behavior:
|
||||
- uploads **new + edited** files
|
||||
- does **not** delete objects from S3
|
||||
- runs on a systemd timer (`services.clawdinator.publicS3.schedule`, default every 10 min)
|
||||
|
||||
Note: current PR intent skill output path is `/memory/pr-intent/...` (on EFS). That matches the default `sourceDir`.
|
||||
|
||||
## Maintainer download (no AWS creds)
|
||||
|
||||
List:
|
||||
|
||||
```sh
|
||||
aws s3 ls s3://<bucket>/ --no-sign-request
|
||||
```
|
||||
|
||||
Pull everything:
|
||||
|
||||
```sh
|
||||
aws s3 sync s3://<bucket>/ ./pr-intent --no-sign-request
|
||||
```
|
||||
@ -13,8 +13,9 @@ export AWS_ACCESS_KEY_ID=...
|
||||
export AWS_SECRET_ACCESS_KEY=...
|
||||
export AWS_REGION=eu-central-1
|
||||
export TF_VAR_aws_region=eu-central-1
|
||||
export TF_VAR_ami_id=ami-... # leave empty to skip instance creation
|
||||
export TF_VAR_ssh_public_key="$(cat ~/.ssh/id_ed25519.pub)" # required when ami_id is set
|
||||
export TF_VAR_manage_instances=true
|
||||
export TF_VAR_ami_id=ami-... # required when manage_instances is true
|
||||
export TF_VAR_ssh_public_key="$(cat ~/.ssh/id_ed25519.pub)" # required when manage_instances is true
|
||||
```
|
||||
|
||||
### Remote state (S3 + Dynamo)
|
||||
@ -44,6 +45,7 @@ export TF_VAR_github_token=...
|
||||
|
||||
## Outputs
|
||||
- `bucket_name`
|
||||
- `pr_intent_bucket_name`
|
||||
- `aws_region`
|
||||
- `ci_user_name`
|
||||
- `access_key_id`
|
||||
|
||||
@ -7,9 +7,12 @@ provider "aws" {
|
||||
}
|
||||
|
||||
locals {
|
||||
tags = merge(var.tags, { "app" = "clawdinator" })
|
||||
tags = merge(var.tags, { "app" = "clawdinator" })
|
||||
instances = jsondecode(file("${path.module}/../../../nix/instances.json"))
|
||||
instance_enabled = var.ami_id != "" && length(local.instances) > 0
|
||||
|
||||
# Safer toggle: instances are managed unless explicitly disabled.
|
||||
# This avoids accidental fleet destruction when TF_VAR_ami_id is omitted.
|
||||
instance_enabled = var.manage_instances && length(local.instances) > 0
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket" "image_bucket" {
|
||||
@ -198,14 +201,14 @@ data "aws_iam_policy_document" "ami_importer" {
|
||||
}
|
||||
|
||||
statement {
|
||||
sid = "PassVmImportRole"
|
||||
actions = ["iam:PassRole"]
|
||||
sid = "PassVmImportRole"
|
||||
actions = ["iam:PassRole"]
|
||||
resources = [aws_iam_role.vmimport.arn]
|
||||
}
|
||||
|
||||
statement {
|
||||
sid = "PassInstanceRole"
|
||||
actions = ["iam:PassRole"]
|
||||
sid = "PassInstanceRole"
|
||||
actions = ["iam:PassRole"]
|
||||
resources = [aws_iam_role.instance.arn]
|
||||
}
|
||||
}
|
||||
@ -430,9 +433,9 @@ resource "aws_iam_role_policy_attachment" "control_lambda_basic" {
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy" "control_lambda_ec2" {
|
||||
count = var.control_api_enabled ? 1 : 0
|
||||
name = "clawdinator-control-ec2"
|
||||
role = aws_iam_role.control_lambda[0].id
|
||||
count = var.control_api_enabled ? 1 : 0
|
||||
name = "clawdinator-control-ec2"
|
||||
role = aws_iam_role.control_lambda[0].id
|
||||
policy = jsonencode({
|
||||
Version = "2012-10-17"
|
||||
Statement = [
|
||||
@ -475,12 +478,12 @@ resource "aws_lambda_function_url" "control" {
|
||||
}
|
||||
|
||||
resource "aws_lambda_permission" "control_url" {
|
||||
count = var.control_api_enabled ? 1 : 0
|
||||
statement_id = "AllowFunctionUrl"
|
||||
action = "lambda:InvokeFunctionUrl"
|
||||
function_name = aws_lambda_function.control[0].function_name
|
||||
principal = "*"
|
||||
function_url_auth_type = "NONE"
|
||||
count = var.control_api_enabled ? 1 : 0
|
||||
statement_id = "AllowFunctionUrl"
|
||||
action = "lambda:InvokeFunctionUrl"
|
||||
function_name = aws_lambda_function.control[0].function_name
|
||||
principal = "*"
|
||||
function_url_auth_type = "NONE"
|
||||
}
|
||||
|
||||
resource "aws_iam_user" "control_invoker" {
|
||||
@ -497,12 +500,12 @@ resource "aws_iam_access_key" "control_invoker" {
|
||||
data "aws_iam_policy_document" "control_invoker" {
|
||||
count = var.control_api_enabled ? 1 : 0
|
||||
statement {
|
||||
actions = ["lambda:InvokeFunction"]
|
||||
actions = ["lambda:InvokeFunction"]
|
||||
resources = [aws_lambda_function.control[0].arn]
|
||||
}
|
||||
|
||||
statement {
|
||||
actions = ["ec2:DescribeInstances"]
|
||||
actions = ["ec2:DescribeInstances"]
|
||||
resources = ["*"]
|
||||
}
|
||||
}
|
||||
|
||||
@ -2,6 +2,11 @@ output "bucket_name" {
|
||||
value = aws_s3_bucket.image_bucket.bucket
|
||||
}
|
||||
|
||||
output "pr_intent_bucket_name" {
|
||||
value = aws_s3_bucket.pr_intent_public.bucket
|
||||
description = "Public S3 bucket for anonymous read/list of PR intent artifacts."
|
||||
}
|
||||
|
||||
output "aws_region" {
|
||||
value = var.aws_region
|
||||
}
|
||||
|
||||
119
infra/opentofu/aws/pr-intent-public-bucket.tf
Normal file
119
infra/opentofu/aws/pr-intent-public-bucket.tf
Normal file
@ -0,0 +1,119 @@
|
||||
data "aws_caller_identity" "current" {}
|
||||
|
||||
locals {
|
||||
pr_intent_bucket_name = var.pr_intent_bucket_name != "" ? var.pr_intent_bucket_name : "openclaw-pr-intent-${data.aws_caller_identity.current.account_id}"
|
||||
}
|
||||
|
||||
# Public bucket hosting PR intent artifacts (anonymous read + list).
|
||||
resource "aws_s3_bucket" "pr_intent_public" {
|
||||
bucket = local.pr_intent_bucket_name
|
||||
tags = local.tags
|
||||
|
||||
lifecycle {
|
||||
prevent_destroy = true
|
||||
}
|
||||
}
|
||||
|
||||
# Allow bucket policy to grant public access (but keep ACL-based public access blocked).
|
||||
resource "aws_s3_bucket_public_access_block" "pr_intent_public" {
|
||||
bucket = aws_s3_bucket.pr_intent_public.id
|
||||
|
||||
block_public_acls = true
|
||||
ignore_public_acls = true
|
||||
block_public_policy = false
|
||||
restrict_public_buckets = false
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_ownership_controls" "pr_intent_public" {
|
||||
bucket = aws_s3_bucket.pr_intent_public.id
|
||||
|
||||
rule {
|
||||
object_ownership = "BucketOwnerEnforced"
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_server_side_encryption_configuration" "pr_intent_public" {
|
||||
bucket = aws_s3_bucket.pr_intent_public.id
|
||||
|
||||
rule {
|
||||
apply_server_side_encryption_by_default {
|
||||
sse_algorithm = "AES256"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_versioning" "pr_intent_public" {
|
||||
bucket = aws_s3_bucket.pr_intent_public.id
|
||||
|
||||
versioning_configuration {
|
||||
status = var.pr_intent_bucket_versioning_enabled ? "Enabled" : "Suspended"
|
||||
}
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "pr_intent_public_bucket_policy" {
|
||||
statement {
|
||||
sid = "AnonymousList"
|
||||
actions = ["s3:ListBucket"]
|
||||
resources = [
|
||||
aws_s3_bucket.pr_intent_public.arn
|
||||
]
|
||||
|
||||
principals {
|
||||
type = "*"
|
||||
identifiers = ["*"]
|
||||
}
|
||||
}
|
||||
|
||||
statement {
|
||||
sid = "AnonymousRead"
|
||||
actions = ["s3:GetObject"]
|
||||
resources = [
|
||||
"${aws_s3_bucket.pr_intent_public.arn}/*"
|
||||
]
|
||||
|
||||
principals {
|
||||
type = "*"
|
||||
identifiers = ["*"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_policy" "pr_intent_public" {
|
||||
bucket = aws_s3_bucket.pr_intent_public.id
|
||||
policy = data.aws_iam_policy_document.pr_intent_public_bucket_policy.json
|
||||
|
||||
depends_on = [aws_s3_bucket_public_access_block.pr_intent_public]
|
||||
}
|
||||
|
||||
# Allow CLAWDINATOR instances to publish artifacts into the public bucket.
|
||||
# (No DeleteObject by default; we publish new/updated files only.)
|
||||
data "aws_iam_policy_document" "instance_pr_intent_publish" {
|
||||
statement {
|
||||
sid = "PublishPrIntentArtifacts"
|
||||
actions = [
|
||||
"s3:PutObject",
|
||||
"s3:PutObjectTagging",
|
||||
"s3:AbortMultipartUpload",
|
||||
"s3:ListBucketMultipartUploads",
|
||||
"s3:ListMultipartUploadParts",
|
||||
"s3:CreateMultipartUpload",
|
||||
"s3:UploadPart",
|
||||
"s3:CompleteMultipartUpload"
|
||||
]
|
||||
resources = [
|
||||
"${aws_s3_bucket.pr_intent_public.arn}/*"
|
||||
]
|
||||
}
|
||||
|
||||
statement {
|
||||
sid = "BucketLocation"
|
||||
actions = ["s3:GetBucketLocation"]
|
||||
resources = [aws_s3_bucket.pr_intent_public.arn]
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy" "instance_pr_intent_publish" {
|
||||
name = "clawdinator-pr-intent-publish"
|
||||
role = aws_iam_role.instance.id
|
||||
policy = data.aws_iam_policy_document.instance_pr_intent_publish.json
|
||||
}
|
||||
@ -9,6 +9,18 @@ variable "bucket_name" {
|
||||
default = "clawdinator-images-eu1-20260107165216"
|
||||
}
|
||||
|
||||
variable "pr_intent_bucket_name" {
|
||||
description = "Public S3 bucket name for PR intent artifacts. Leave empty to derive a per-account default (openclaw-pr-intent-<account_id>)."
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
|
||||
variable "pr_intent_bucket_versioning_enabled" {
|
||||
description = "Enable S3 versioning for the public PR intent bucket (useful while iterating on outputs)."
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "ci_user_name" {
|
||||
description = "IAM user used by CI."
|
||||
type = string
|
||||
@ -21,10 +33,20 @@ variable "tags" {
|
||||
default = {}
|
||||
}
|
||||
|
||||
variable "manage_instances" {
|
||||
description = "Whether to manage (create/update/destroy) the CLAWDINATOR EC2 instances and related networking resources."
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "ami_id" {
|
||||
description = "AMI ID for CLAWDINATOR instances."
|
||||
type = string
|
||||
default = ""
|
||||
validation {
|
||||
condition = !var.manage_instances || var.ami_id != ""
|
||||
error_message = "ami_id is required when manage_instances is true."
|
||||
}
|
||||
}
|
||||
|
||||
variable "root_volume_size_gb" {
|
||||
@ -38,8 +60,8 @@ variable "ssh_public_key" {
|
||||
type = string
|
||||
default = ""
|
||||
validation {
|
||||
condition = var.ami_id == "" || length(var.ssh_public_key) > 0
|
||||
error_message = "ssh_public_key is required when ami_id is set."
|
||||
condition = !var.manage_instances || length(var.ssh_public_key) > 0
|
||||
error_message = "ssh_public_key is required when manage_instances is true."
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -21,4 +21,14 @@
|
||||
|
||||
networking.firewall.allowedTCPPorts = [ 22 ];
|
||||
|
||||
# Publish PR intent artifacts from EFS to the public bucket.
|
||||
# (Timer + oneshot service; safe to run without stopping the gateway.)
|
||||
services.clawdinator.publicS3 = {
|
||||
enable = true;
|
||||
bucket = "openclaw-pr-intent";
|
||||
region = "eu-central-1";
|
||||
sourceDir = "/memory/pr-intent";
|
||||
# schedule = "*:0/10"; # default
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -128,6 +128,9 @@ in
|
||||
primary = "openai/gpt-5.2-codex";
|
||||
fallbacks = [ "anthropic/claude-opus-4-6" ];
|
||||
};
|
||||
|
||||
# Default thinking level for reasoning-capable models (GPT-5.2/Codex).
|
||||
thinkingDefault = "high";
|
||||
};
|
||||
agents.list = [
|
||||
{
|
||||
|
||||
@ -441,6 +441,47 @@ in
|
||||
description = "GitHub org to sync.";
|
||||
};
|
||||
};
|
||||
|
||||
publicS3 = {
|
||||
enable = mkEnableOption "Publish a public-safe subtree from /memory to a public S3 bucket";
|
||||
|
||||
schedule = mkOption {
|
||||
type = types.str;
|
||||
default = "*:0/10";
|
||||
description = "systemd OnCalendar schedule for public S3 publish (default: every 10 min).";
|
||||
};
|
||||
|
||||
region = mkOption {
|
||||
type = types.str;
|
||||
default = "eu-central-1";
|
||||
description = "AWS region for the public S3 bucket.";
|
||||
};
|
||||
|
||||
bucket = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
description = "Destination S3 bucket name (public-read/list bucket).";
|
||||
};
|
||||
|
||||
# Keep this narrow and explicit: publish only the public-safe subtree.
|
||||
sourceDir = mkOption {
|
||||
type = types.str;
|
||||
default = "${cfg.memoryDir}/pr-intent";
|
||||
description = "Local directory tree to publish to S3 (should contain only public-safe files).";
|
||||
};
|
||||
|
||||
destPrefix = mkOption {
|
||||
type = types.str;
|
||||
default = "";
|
||||
description = "Optional S3 key prefix under the bucket (leave empty to publish at bucket root).";
|
||||
};
|
||||
|
||||
stateDir = mkOption {
|
||||
type = types.str;
|
||||
default = "${cfg.stateDir}/public-s3";
|
||||
description = "State directory for public S3 publishing (stamp + lock).";
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
@ -468,6 +509,10 @@ in
|
||||
assertion = (!cfg.memoryEfs.enable) || (cfg.memoryEfs.fileSystemId != "");
|
||||
message = "services.clawdinator.memoryEfs requires fileSystemId.";
|
||||
}
|
||||
{
|
||||
assertion = (!cfg.publicS3.enable) || (cfg.publicS3.bucket != "");
|
||||
message = "services.clawdinator.publicS3 requires bucket.";
|
||||
}
|
||||
];
|
||||
|
||||
users.groups.${cfg.group} = {};
|
||||
@ -590,25 +635,27 @@ in
|
||||
]
|
||||
);
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d ${cfg.stateDir} 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"d ${cfg.stateDir}/.pi 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"d ${cfg.stateDir}/.pi/agent 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"L+ ${cfg.stateDir}/.pi/agent/settings.json - - - - /etc/clawdinator/pi-settings.json"
|
||||
"d ${workspaceDir} 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"d ${logDir} 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"d ${ghConfigDir} 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"d /run/clawd 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"z /run/clawd 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"f /run/clawd/github-app.env 0640 ${cfg.user} ${cfg.group} - -"
|
||||
"z /run/clawd/github-app.env 0640 ${cfg.user} ${cfg.group} - -"
|
||||
"d ${cfg.memoryDir} 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"d ${repoSeedBaseDir} 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"d /usr/local/bin 0755 root root - -"
|
||||
"L+ /usr/local/bin/memory-read - - - - /etc/clawdinator/bin/memory-read"
|
||||
"L+ /usr/local/bin/memory-write - - - - /etc/clawdinator/bin/memory-write"
|
||||
"L+ /usr/local/bin/memory-edit - - - - /etc/clawdinator/bin/memory-edit"
|
||||
];
|
||||
systemd.tmpfiles.rules =
|
||||
[
|
||||
"d ${cfg.stateDir} 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"d ${cfg.stateDir}/.pi 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"d ${cfg.stateDir}/.pi/agent 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"L+ ${cfg.stateDir}/.pi/agent/settings.json - - - - /etc/clawdinator/pi-settings.json"
|
||||
"d ${workspaceDir} 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"d ${logDir} 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"d ${ghConfigDir} 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"d /run/clawd 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"z /run/clawd 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"f /run/clawd/github-app.env 0640 ${cfg.user} ${cfg.group} - -"
|
||||
"z /run/clawd/github-app.env 0640 ${cfg.user} ${cfg.group} - -"
|
||||
"d ${cfg.memoryDir} 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"d ${repoSeedBaseDir} 0750 ${cfg.user} ${cfg.group} - -"
|
||||
"d /usr/local/bin 0755 root root - -"
|
||||
"L+ /usr/local/bin/memory-read - - - - /etc/clawdinator/bin/memory-read"
|
||||
"L+ /usr/local/bin/memory-write - - - - /etc/clawdinator/bin/memory-write"
|
||||
"L+ /usr/local/bin/memory-edit - - - - /etc/clawdinator/bin/memory-edit"
|
||||
]
|
||||
++ lib.optional cfg.publicS3.enable "d ${cfg.publicS3.stateDir} 0750 ${cfg.user} ${cfg.group} - -";
|
||||
|
||||
fileSystems = lib.mkIf cfg.memoryEfs.enable {
|
||||
"${cfg.memoryEfs.mountPoint}" = {
|
||||
@ -853,5 +900,39 @@ in
|
||||
Persistent = true;
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.clawdinator-public-s3-publish = lib.mkIf cfg.publicS3.enable {
|
||||
description = "CLAWDINATOR public S3 publish (mirror public-safe memory subtree)";
|
||||
after =
|
||||
[ "network-online.target" ]
|
||||
++ lib.optional cfg.memoryEfs.enable "remote-fs.target"
|
||||
++ lib.optional cfg.memoryEfs.enable "clawdinator-memory-init.service";
|
||||
wants = [ "network-online.target" ] ++ lib.optional cfg.memoryEfs.enable "remote-fs.target";
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
User = cfg.user;
|
||||
Group = cfg.group;
|
||||
};
|
||||
environment = {
|
||||
AWS_REGION = cfg.publicS3.region;
|
||||
AWS_DEFAULT_REGION = cfg.publicS3.region;
|
||||
};
|
||||
path = [ pkgs.awscli2 pkgs.coreutils pkgs.findutils pkgs.util-linux ];
|
||||
script = ''
|
||||
exec ${../../scripts/sync-public-s3-tree.sh} \
|
||||
${lib.escapeShellArg cfg.publicS3.sourceDir} \
|
||||
${lib.escapeShellArg cfg.publicS3.bucket} \
|
||||
${lib.escapeShellArg cfg.publicS3.destPrefix} \
|
||||
${lib.escapeShellArg cfg.publicS3.stateDir}
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.clawdinator-public-s3-publish = lib.mkIf cfg.publicS3.enable {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnCalendar = cfg.publicS3.schedule;
|
||||
Persistent = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
73
scripts/sync-public-s3-tree.sh
Executable file
73
scripts/sync-public-s3-tree.sh
Executable file
@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
source_dir="${1:-}"
|
||||
bucket="${2:-}"
|
||||
dest_prefix="${3:-}"
|
||||
state_dir="${4:-}"
|
||||
|
||||
if [ -z "$source_dir" ] || [ -z "$bucket" ] || [ -z "$state_dir" ]; then
|
||||
echo "Usage: sync-public-s3-tree.sh <source_dir> <bucket> <dest_prefix> <state_dir>" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Normalize prefix: allow empty or trailing slash.
|
||||
if [ -n "$dest_prefix" ] && [[ "$dest_prefix" != */ ]]; then
|
||||
dest_prefix="${dest_prefix}/"
|
||||
fi
|
||||
|
||||
if [ ! -d "$source_dir" ]; then
|
||||
echo "sync-public-s3-tree: source dir missing; nothing to do: $source_dir" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "$state_dir"
|
||||
|
||||
lock_file="$state_dir/sync.lock"
|
||||
stamp_file="$state_dir/last-success.stamp"
|
||||
|
||||
if [ ! -f "$stamp_file" ]; then
|
||||
# Epoch-ish, so first run uploads everything.
|
||||
touch -t 197001010000 "$stamp_file"
|
||||
fi
|
||||
|
||||
exec 9>"$lock_file"
|
||||
if ! flock -n 9; then
|
||||
# Another run is in progress.
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Mark the start time; anything modified after this will be picked up next run.
|
||||
run_stamp="$state_dir/run.stamp"
|
||||
touch "$run_stamp"
|
||||
|
||||
# Find files newer than the last successful run, but not newer than this run's start.
|
||||
# (Prevents missing files that are created/modified during the upload.)
|
||||
mapfile -d '' files < <(find "$source_dir" -type f -newer "$stamp_file" ! -newer "$run_stamp" -print0)
|
||||
|
||||
if [ "${#files[@]}" -eq 0 ]; then
|
||||
# Nothing to upload; still advance the stamp.
|
||||
mv -f "$run_stamp" "$stamp_file"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
for f in "${files[@]}"; do
|
||||
rel="${f#"$source_dir"/}"
|
||||
if [ "$rel" = "$f" ]; then
|
||||
# Shouldn't happen, but be safe.
|
||||
echo "sync-public-s3-tree: failed to compute relative path for $f" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Use path-style keys; preserve directory structure.
|
||||
dst="s3://${bucket}/${dest_prefix}${rel}"
|
||||
|
||||
# Overwrite is allowed (iteration mode). No deletes.
|
||||
aws s3 cp \
|
||||
--only-show-errors \
|
||||
--no-progress \
|
||||
"$f" \
|
||||
"$dst"
|
||||
done
|
||||
|
||||
mv -f "$run_stamp" "$stamp_file"
|
||||
Loading…
Reference in New Issue
Block a user