mirror of https://github.com/garrytan/gstack.git
feat(gbrain-sync): queue primitives + writer shims
Adds bin/gstack-brain-enqueue (atomic append to sync queue) and bin/gstack-jsonl-merge (git merge driver, ts-sort with SHA-256 fallback). Wires one backgrounded enqueue call into learnings-log, timeline-log, review-log, and developer-profile --migrate. question-log and question-preferences stay local per Codex v2 decision. gstack-config gains gbrain_sync_mode (off/artifacts-only/full) and gbrain_sync_mode_prompted keys, plus GSTACK_HOME env alignment so tests don't leak into real ~/.gstack/config.yaml.
This commit is contained in:
parent
656df0e37e
commit
45638297ba
|
|
@ -0,0 +1,55 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# gstack-brain-enqueue — atomically append a path to the GBrain sync queue.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# gstack-brain-enqueue <file-path>
|
||||||
|
#
|
||||||
|
# Called by writer scripts (gstack-learnings-log, gstack-timeline-log, etc.)
|
||||||
|
# after their local write. Fire-and-forget; failures are silent (never blocks
|
||||||
|
# the writer). Queue is drained by `gstack-brain-sync --once` invoked from the
|
||||||
|
# preamble at skill START and END boundaries.
|
||||||
|
#
|
||||||
|
# No-op when:
|
||||||
|
# - gbrain_sync_mode is off (the default)
|
||||||
|
# - ~/.gstack/.git doesn't exist (feature not initialized)
|
||||||
|
# - <file-path> matches a line in ~/.gstack/.brain-skip.txt
|
||||||
|
#
|
||||||
|
# Env:
|
||||||
|
# GSTACK_HOME — override ~/.gstack state directory (aligns with writers).
|
||||||
|
# Tests use GSTACK_HOME=/tmp/test-$$ for isolation.
|
||||||
|
#
|
||||||
|
# Concurrency: POSIX append is atomic up to PIPE_BUF (~4KB Linux, 512 BSD).
|
||||||
|
# Queue lines are ~200 bytes, safe under concurrent callers.
|
||||||
|
|
||||||
|
# No `-e` — writer shims rely on this never failing loudly.
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
FILE="${1:-}"
|
||||||
|
[ -z "$FILE" ] && exit 0
|
||||||
|
|
||||||
|
GSTACK_HOME="${GSTACK_HOME:-$HOME/.gstack}"
|
||||||
|
QUEUE="$GSTACK_HOME/.brain-queue.jsonl"
|
||||||
|
SKIP_FILE="$GSTACK_HOME/.brain-skip.txt"
|
||||||
|
|
||||||
|
# Fast exits: no git repo, no sync.
|
||||||
|
[ ! -d "$GSTACK_HOME/.git" ] && exit 0
|
||||||
|
|
||||||
|
# Check sync mode. off → silent no-op.
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" 2>/dev/null && pwd)"
|
||||||
|
MODE=$("$SCRIPT_DIR/gstack-config" get gbrain_sync_mode 2>/dev/null || echo off)
|
||||||
|
[ "$MODE" = "off" ] && exit 0
|
||||||
|
|
||||||
|
# User-maintained skip list (for secret-scan false positives).
|
||||||
|
if [ -f "$SKIP_FILE" ]; then
|
||||||
|
if grep -Fxq "$FILE" "$SKIP_FILE" 2>/dev/null; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# JSON-escape the file path (backslash + quotes only; paths shouldn't have other specials).
|
||||||
|
ESC_FILE=$(printf '%s' "$FILE" | sed 's/\\/\\\\/g; s/"/\\"/g')
|
||||||
|
TS=$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo "")
|
||||||
|
|
||||||
|
printf '{"file":"%s","ts":"%s"}\n' "$ESC_FILE" "$TS" >> "$QUEUE" 2>/dev/null
|
||||||
|
|
||||||
|
exit 0
|
||||||
|
|
@ -8,10 +8,11 @@
|
||||||
# gstack-config defaults — show just the defaults table
|
# gstack-config defaults — show just the defaults table
|
||||||
#
|
#
|
||||||
# Env overrides (for testing):
|
# Env overrides (for testing):
|
||||||
# GSTACK_STATE_DIR — override ~/.gstack state directory
|
# GSTACK_HOME — override ~/.gstack state directory (aligns with writer scripts)
|
||||||
|
# GSTACK_STATE_DIR — legacy alias for GSTACK_HOME (kept for backwards compat)
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
STATE_DIR="${GSTACK_STATE_DIR:-$HOME/.gstack}"
|
STATE_DIR="${GSTACK_HOME:-${GSTACK_STATE_DIR:-$HOME/.gstack}}"
|
||||||
CONFIG_FILE="$STATE_DIR/config.yaml"
|
CONFIG_FILE="$STATE_DIR/config.yaml"
|
||||||
|
|
||||||
# Annotated header for new config files. Written once on first `set`.
|
# Annotated header for new config files. Written once on first `set`.
|
||||||
|
|
@ -59,6 +60,19 @@ CONFIG_HEADER='# gstack configuration — edit freely, changes take effect on ne
|
||||||
# # Unknown values default to "default" with a warning.
|
# # Unknown values default to "default" with a warning.
|
||||||
# # See docs/designs/PLAN_TUNING_V1.md for rationale.
|
# # See docs/designs/PLAN_TUNING_V1.md for rationale.
|
||||||
#
|
#
|
||||||
|
# ─── GBrain sync (v1.7+) ─────────────────────────────────────────────
|
||||||
|
# gbrain_sync_mode: off # off | artifacts-only | full
|
||||||
|
# # off — no sync (default)
|
||||||
|
# # artifacts-only — sync plans/designs/retros/learnings only
|
||||||
|
# # (skip behavioral data: question-log,
|
||||||
|
# # developer-profile, timeline)
|
||||||
|
# # full — sync everything allowlisted
|
||||||
|
# # Set by the first-run privacy stop-gate. See docs/gbrain-sync.md.
|
||||||
|
#
|
||||||
|
# gbrain_sync_mode_prompted: false
|
||||||
|
# # Set to true once the privacy gate has asked the user.
|
||||||
|
# # Flip back to false to be re-prompted.
|
||||||
|
#
|
||||||
# ─── Advanced ────────────────────────────────────────────────────────
|
# ─── Advanced ────────────────────────────────────────────────────────
|
||||||
# codex_reviews: enabled # disabled = skip Codex adversarial reviews in /ship
|
# codex_reviews: enabled # disabled = skip Codex adversarial reviews in /ship
|
||||||
# gstack_contributor: false # true = file field reports when gstack misbehaves
|
# gstack_contributor: false # true = file field reports when gstack misbehaves
|
||||||
|
|
@ -83,6 +97,8 @@ lookup_default() {
|
||||||
gstack_contributor) echo "false" ;;
|
gstack_contributor) echo "false" ;;
|
||||||
skip_eng_review) echo "false" ;;
|
skip_eng_review) echo "false" ;;
|
||||||
cross_project_learnings) echo "" ;; # intentionally empty → unset triggers first-time prompt
|
cross_project_learnings) echo "" ;; # intentionally empty → unset triggers first-time prompt
|
||||||
|
gbrain_sync_mode) echo "off" ;;
|
||||||
|
gbrain_sync_mode_prompted) echo "false" ;;
|
||||||
*) echo "" ;;
|
*) echo "" ;;
|
||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
|
|
@ -114,6 +130,10 @@ case "${1:-}" in
|
||||||
echo "Warning: explain_level '$VALUE' not recognized. Valid values: default, terse. Using default." >&2
|
echo "Warning: explain_level '$VALUE' not recognized. Valid values: default, terse. Using default." >&2
|
||||||
VALUE="default"
|
VALUE="default"
|
||||||
fi
|
fi
|
||||||
|
if [ "$KEY" = "gbrain_sync_mode" ] && [ "$VALUE" != "off" ] && [ "$VALUE" != "artifacts-only" ] && [ "$VALUE" != "full" ]; then
|
||||||
|
echo "Warning: gbrain_sync_mode '$VALUE' not recognized. Valid values: off, artifacts-only, full. Using off." >&2
|
||||||
|
VALUE="off"
|
||||||
|
fi
|
||||||
mkdir -p "$STATE_DIR"
|
mkdir -p "$STATE_DIR"
|
||||||
# Write annotated header on first creation
|
# Write annotated header on first creation
|
||||||
if [ ! -f "$CONFIG_FILE" ]; then
|
if [ ! -f "$CONFIG_FILE" ]; then
|
||||||
|
|
@ -142,7 +162,8 @@ case "${1:-}" in
|
||||||
echo "# ─── Active values (including defaults for unset keys) ───"
|
echo "# ─── Active values (including defaults for unset keys) ───"
|
||||||
for KEY in proactive routing_declined telemetry auto_upgrade update_check \
|
for KEY in proactive routing_declined telemetry auto_upgrade update_check \
|
||||||
skill_prefix checkpoint_mode checkpoint_push codex_reviews \
|
skill_prefix checkpoint_mode checkpoint_push codex_reviews \
|
||||||
gstack_contributor skip_eng_review; do
|
gstack_contributor skip_eng_review gbrain_sync_mode \
|
||||||
|
gbrain_sync_mode_prompted; do
|
||||||
VALUE=$(grep -E "^${KEY}:" "$CONFIG_FILE" 2>/dev/null | tail -1 | awk '{print $2}' | tr -d '[:space:]' || true)
|
VALUE=$(grep -E "^${KEY}:" "$CONFIG_FILE" 2>/dev/null | tail -1 | awk '{print $2}' | tr -d '[:space:]' || true)
|
||||||
SOURCE="default"
|
SOURCE="default"
|
||||||
if [ -n "$VALUE" ]; then
|
if [ -n "$VALUE" ]; then
|
||||||
|
|
@ -157,7 +178,8 @@ case "${1:-}" in
|
||||||
echo "# gstack-config defaults"
|
echo "# gstack-config defaults"
|
||||||
for KEY in proactive routing_declined telemetry auto_upgrade update_check \
|
for KEY in proactive routing_declined telemetry auto_upgrade update_check \
|
||||||
skill_prefix checkpoint_mode checkpoint_push codex_reviews \
|
skill_prefix checkpoint_mode checkpoint_push codex_reviews \
|
||||||
gstack_contributor skip_eng_review; do
|
gstack_contributor skip_eng_review gbrain_sync_mode \
|
||||||
|
gbrain_sync_mode_prompted; do
|
||||||
printf ' %-24s %s\n' "$KEY:" "$(lookup_default "$KEY")"
|
printf ' %-24s %s\n' "$KEY:" "$(lookup_default "$KEY")"
|
||||||
done
|
done
|
||||||
;;
|
;;
|
||||||
|
|
|
||||||
|
|
@ -101,6 +101,10 @@ do_migrate() {
|
||||||
mv "$TMPOUT" "$PROFILE_FILE"
|
mv "$TMPOUT" "$PROFILE_FILE"
|
||||||
trap - EXIT
|
trap - EXIT
|
||||||
|
|
||||||
|
# gbrain-sync: enqueue the migrated file for cross-machine sync (no-op if off).
|
||||||
|
SCRIPT_DIR_E="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
"$SCRIPT_DIR_E/gstack-brain-enqueue" "developer-profile.json" 2>/dev/null &
|
||||||
|
|
||||||
# Archive the legacy file.
|
# Archive the legacy file.
|
||||||
local TS
|
local TS
|
||||||
TS="$(date +%Y-%m-%d-%H%M%S)"
|
TS="$(date +%Y-%m-%d-%H%M%S)"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,88 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# gstack-jsonl-merge — git merge driver for append-only JSONL files.
|
||||||
|
#
|
||||||
|
# Usage (called by git, not by users):
|
||||||
|
# gstack-jsonl-merge <base> <ours> <theirs>
|
||||||
|
#
|
||||||
|
# Registered in local git config by bin/gstack-brain-init and
|
||||||
|
# bin/gstack-brain-restore:
|
||||||
|
# git config merge.jsonl-append.driver \
|
||||||
|
# "$GSTACK_BIN/gstack-jsonl-merge %O %A %B"
|
||||||
|
#
|
||||||
|
# Behavior:
|
||||||
|
# Concatenate base + ours + theirs, dedup exact-duplicate lines, sort by
|
||||||
|
# ISO "ts" field when present, fall back to SHA-256 of the line for
|
||||||
|
# deterministic order. Write result to <ours> (the %A file per the git
|
||||||
|
# merge-driver contract).
|
||||||
|
#
|
||||||
|
# Two machines appending to the same JSONL file between pushes produces
|
||||||
|
# a same-line conflict at the file tail. This driver resolves it cleanly:
|
||||||
|
# both appends survive, ordered by wall-clock timestamp where available,
|
||||||
|
# content hash otherwise.
|
||||||
|
#
|
||||||
|
# Exit codes:
|
||||||
|
# 0 — merge succeeded, result written to <ours>
|
||||||
|
# 1 — error; git treats as conflict and stops the merge
|
||||||
|
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
if [ "$#" -lt 3 ]; then
|
||||||
|
echo "gstack-jsonl-merge: expected 3 args (base ours theirs), got $#" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
BASE="$1"
|
||||||
|
OURS="$2"
|
||||||
|
THEIRS="$3"
|
||||||
|
|
||||||
|
TMP=$(mktemp /tmp/gstack-jsonl-merge.XXXXXX) || exit 1
|
||||||
|
trap 'rm -f "$TMP" 2>/dev/null || true' EXIT
|
||||||
|
|
||||||
|
python3 - "$BASE" "$OURS" "$THEIRS" > "$TMP" <<'PYEOF'
|
||||||
|
import sys, json, hashlib
|
||||||
|
|
||||||
|
paths = sys.argv[1:4] # base, ours, theirs
|
||||||
|
seen = {} # line content -> sort_key
|
||||||
|
|
||||||
|
for path in paths:
|
||||||
|
try:
|
||||||
|
with open(path, 'r', encoding='utf-8') as f:
|
||||||
|
for line in f:
|
||||||
|
line = line.rstrip('\n')
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
if line in seen:
|
||||||
|
continue
|
||||||
|
# Prefer ISO ts field for sort; fall back to SHA-256.
|
||||||
|
sort_key = None
|
||||||
|
try:
|
||||||
|
obj = json.loads(line)
|
||||||
|
ts = obj.get('ts') or obj.get('timestamp')
|
||||||
|
if isinstance(ts, str):
|
||||||
|
sort_key = (0, ts)
|
||||||
|
except (json.JSONDecodeError, ValueError, TypeError):
|
||||||
|
pass
|
||||||
|
if sort_key is None:
|
||||||
|
h = hashlib.sha256(line.encode('utf-8')).hexdigest()
|
||||||
|
sort_key = (1, h)
|
||||||
|
seen[line] = sort_key
|
||||||
|
except FileNotFoundError:
|
||||||
|
# Absent base / absent ours / absent theirs are all valid.
|
||||||
|
continue
|
||||||
|
except OSError:
|
||||||
|
# Permission / IO errors are fatal — caller sees non-zero exit.
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Timestamp-ordered entries first (group 0), then hash-ordered (group 1).
|
||||||
|
for line, _ in sorted(seen.items(), key=lambda item: item[1]):
|
||||||
|
print(line)
|
||||||
|
PYEOF
|
||||||
|
|
||||||
|
_PYEXIT=$?
|
||||||
|
if [ "$_PYEXIT" != "0" ]; then
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
mv "$TMP" "$OURS" || exit 1
|
||||||
|
trap - EXIT
|
||||||
|
exit 0
|
||||||
|
|
@ -84,3 +84,6 @@ if [ $? -ne 0 ] || [ -z "$VALIDATED" ]; then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "$VALIDATED" >> "$GSTACK_HOME/projects/$SLUG/learnings.jsonl"
|
echo "$VALIDATED" >> "$GSTACK_HOME/projects/$SLUG/learnings.jsonl"
|
||||||
|
|
||||||
|
# gbrain-sync: enqueue for cross-machine sync (no-op if sync is off).
|
||||||
|
"$SCRIPT_DIR/gstack-brain-enqueue" "projects/$SLUG/learnings.jsonl" 2>/dev/null &
|
||||||
|
|
|
||||||
|
|
@ -165,3 +165,7 @@ if [ $VALIDATE_RC -ne 0 ] || [ -z "$VALIDATED" ]; then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "$VALIDATED" >> "$GSTACK_HOME/projects/$SLUG/question-log.jsonl"
|
echo "$VALIDATED" >> "$GSTACK_HOME/projects/$SLUG/question-log.jsonl"
|
||||||
|
|
||||||
|
# NOTE: question-log.jsonl is deliberately NOT enqueued for gbrain-sync.
|
||||||
|
# Per Codex v2 review, audit/derivation data stays local alongside the
|
||||||
|
# question-preferences.json it annotates.
|
||||||
|
|
|
||||||
|
|
@ -16,3 +16,6 @@ if ! printf '%s' "$INPUT" | bun -e "JSON.parse(await Bun.stdin.text())" 2>/dev/n
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "$INPUT" >> "$GSTACK_HOME/projects/$SLUG/$BRANCH-reviews.jsonl"
|
echo "$INPUT" >> "$GSTACK_HOME/projects/$SLUG/$BRANCH-reviews.jsonl"
|
||||||
|
|
||||||
|
# gbrain-sync: enqueue for cross-machine sync (no-op if sync is off).
|
||||||
|
"$SCRIPT_DIR/gstack-brain-enqueue" "projects/$SLUG/$BRANCH-reviews.jsonl" 2>/dev/null &
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,10 @@
|
||||||
# gstack-timeline-log — append a timeline event to the project timeline
|
# gstack-timeline-log — append a timeline event to the project timeline
|
||||||
# Usage: gstack-timeline-log '{"skill":"review","event":"started","branch":"main"}'
|
# Usage: gstack-timeline-log '{"skill":"review","event":"started","branch":"main"}'
|
||||||
#
|
#
|
||||||
# Session timeline: local-only, never sent anywhere.
|
# Session timeline: local by default. If the user enables `gbrain_sync_mode`
|
||||||
|
# with the `full` (not `artifacts-only`) privacy tier — via the first-run
|
||||||
|
# stop-gate from `gstack-brain-init` or the preamble — timeline events are
|
||||||
|
# published to the user's private GBrain sync repo. See docs/gbrain-sync.md.
|
||||||
# Required fields: skill, event (started|completed).
|
# Required fields: skill, event (started|completed).
|
||||||
# Optional: branch, outcome, duration_s, session, ts.
|
# Optional: branch, outcome, duration_s, session, ts.
|
||||||
# Validation failure → skip silently (non-blocking).
|
# Validation failure → skip silently (non-blocking).
|
||||||
|
|
@ -32,3 +35,6 @@ if ! printf '%s' "$INPUT" | bun -e "const j=JSON.parse(await Bun.stdin.text());
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "$INPUT" >> "$GSTACK_HOME/projects/$SLUG/timeline.jsonl"
|
echo "$INPUT" >> "$GSTACK_HOME/projects/$SLUG/timeline.jsonl"
|
||||||
|
|
||||||
|
# gbrain-sync: enqueue for cross-machine sync (no-op if sync is off).
|
||||||
|
"$SCRIPT_DIR/gstack-brain-enqueue" "projects/$SLUG/timeline.jsonl" 2>/dev/null &
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue