{ "ok": true, "world": "world-models", "count": 34, "terms": [ { "slug": "active-inference", "term": "Active inference (free-energy principle)", "category": "methods-frameworks", "short": "A variational framework where perception and action both minimize expected free energy, unifying curiosity and goal-seeking.", "definition": "Friston's active inference framework: an agent maintains a generative model of the world and acts to minimize expected free energy, which decomposes into epistemic value (expected information gain — curiosity) and pragmatic value (expected reward / goal proximity). Unlike ICM or WorldLLM, curiosity emerges from the variational objective rather than being a separately engineered signal. Active inference is the mechanism underlying the active-inference-agent system.", "example": "An active-inference agent perceiving an ambiguous scene both acts to disambiguate (epistemic — curiosity) and to achieve its goal (pragmatic) in a single variational computation, with no separate curiosity module.", "related": [ "free-energy-minimization", "information-gain", "bayesian-inference", "active-inference-agent", "intrinsic-motivation" ], "source": "Friston — Active inference & free-energy principle (DeepMind/UCL canonical write-ups); Murphy (variational inference)" }, { "slug": "active-inference-agent", "term": "Active-inference agent", "category": "world-models", "short": "An agent that minimizes expected free energy, balancing epistemic value (information gain) and pragmatic value (goal achievement).", "definition": "An agent architecture based on the free-energy principle (Friston): actions are selected to minimize expected free energy, which decomposes into epistemic value (expected information gain — curiosity) and pragmatic value (expected reward / goal proximity). Active inference unifies perception, action, and curiosity under one variational objective. It differs from ICM/WorldLLM in deriving curiosity from the variational free-energy objective rather than from a prediction-error or log-likelihood signal.", "example": "An active-inference agent in an unknown environment explores regions with high epistemic value (uncertainty) while steering toward goal states — curiosity and goal-seeking emerge from the same variational objective.", "related": [ "active-inference", "free-energy-minimization", "information-gain", "intrinsic-motivation", "world-model" ], "source": "Friston — Active inference & free-energy principle (DeepMind/UCL canonical write-ups); Murphy (variational inference)" }, { "slug": "bayesian-inference", "term": "Bayesian inference", "category": "methods-frameworks", "short": "Updating a probability distribution over hypotheses (the posterior) when new evidence arrives, using Bayes' rule.", "definition": "The general mechanism: Bayes' rule combines a prior over hypotheses P(H) with a likelihood P(evidence | H) to produce a posterior P(H | evidence). Bayesian inference is the general anchor for both Bayesian surprise (the signal) and the Theorist (which performs Bayesian NL hypothesis refinement). It is NOT a learning algorithm — it is the mathematical framework underlying both active inference and the WorldLLM theory-update step.", "example": "A doctor updates the probability that a patient has disease D after seeing a positive test result: P(D | positive) ∝ P(positive | D) × P(D). The Theorist does the analogous update over NL theories of environment dynamics.", "related": [ "hypothesis-refinement", "proposal-distribution", "bayesian-surprise", "information-gain", "active-inference" ], "source": "Murphy — Probabilistic Machine Learning ch. 2 (Bayesian inference, posterior update)" }, { "slug": "bayesian-surprise", "term": "Bayesian surprise", "category": "curiosity-signals", "short": "The KL divergence between the agent's posterior over models before and after an observation.", "definition": "A *different instance* of intrinsic motivation from prediction-error curiosity: surprise measured as how much an observation moves the agent's posterior over models (KL[posterior‖prior]) — i.e. information gained about the world's structure. Because it rewards posterior *change*, not raw unpredictability, it avoids the noisy-TV trap. In WorldLLM this is the signal the Experimenter maximizes — it seeks transitions the current theory finds surprising (low log-likelihood). It is contrasted with prediction-error curiosity: same word 'surprise', different math.", "example": "Two observations are equally unpredictable, but only the one that shifts the agent's belief over which dynamics model is correct yields high Bayesian surprise — the other is just noise.", "related": [ "intrinsic-motivation", "prediction-error-curiosity", "information-gain", "surprising-transition", "bayesian-inference", "epistemic-vs-aleatoric" ], "source": "Murphy — Probabilistic Machine Learning (Bayesian inference / surprise); Latent Bayesian Surprise arXiv:2104.07495" }, { "slug": "boxinggym", "term": "BoxingGym", "category": "world-models", "short": "A benchmark for automated experimental design and model discovery, where an agent proposes experiments and updates a model of the world.", "definition": "Gandhi et al.'s BoxingGym (arXiv:2501.01540): a benchmark environment for evaluating automated experimental design and theory-making. An agent proposes experiments, observes outcomes, and updates a probabilistic model of the world — a formal test bed for hypothesis-refinement machinery like the WorldLLM Theorist loop. It operationalizes the scientist-as-agent framing and provides a quantitative evaluation framework for curiosity-driven model discovery.", "example": "A BoxingGym agent designs minimal experiments to distinguish competing theories of a physical system, selecting the most informative action at each step — a formal analog of the WorldLLM Experimenter's role.", "related": [ "hypothesis-refinement", "worldllm", "information-gain", "bayesian-inference" ], "source": "BoxingGym arXiv:2501.01540; Murphy (Bayesian inference / experimental design)" }, { "slug": "detachment", "term": "Detachment (reward hacking via learned features)", "category": "failure-modes", "short": "A curiosity agent finds ways to maximize its intrinsic reward without encountering genuinely novel or informative states.", "definition": "A failure mode distinct from the noisy-TV problem: the agent learns to maximize the curiosity signal through adversarial exploitation of the feature space or model artifacts rather than by genuinely exploring novel states. For example, in ICM the agent may find feature-space transitions that are high-error without corresponding to novel environment states. It is a generalization of reward hacking to intrinsic-reward systems.", "example": "An ICM agent learns that rapidly toggling between two game states confuses its forward model into high prediction error — without ever discovering new areas of the environment.", "related": [ "intrinsic-reward-hacking", "prediction-error-curiosity", "noisy-tv-problem", "intrinsic-motivation" ], "source": "Pathak ICM arXiv:1705.05363 (limitations); Sutton & Barto (reward hacking framing)" }, { "slug": "dreamer", "term": "DreamerV3", "category": "world-models", "short": "A latent world-model agent that learns an environment model in a compact learned embedding space and trains its policy on imagined rollouts.", "definition": "Hafner et al.'s DreamerV3 (arXiv:2301.04104, building on arXiv:1803.10122): a model-based RL agent that learns a Recurrent State Space Model (RSSM) as a latent world model and trains a policy entirely on imagined trajectories within it. The world model is a latent embedding, not a natural-language theory — which is the key contrast with WorldLLM's Theorist. Dreamer uses extrinsic reward and does not use curiosity-driven exploration by default; it enters this World as a world-model predictor, not as a curiosity system.", "example": "DreamerV3 achieves superhuman Atari scores and positive rewards in Minecraft without human demonstrations by planning entirely in its latent world model, never sampling real environment rollouts during policy learning.", "related": [ "world-model", "intrinsic-motivation", "worldllm", "jepa" ], "source": "DreamerV3 arXiv:2301.04104; Ha & Schmidhuber World Models arXiv:1803.10122; Sutton & Barto (model-based RL)" }, { "slug": "epistemic-vs-aleatoric", "term": "Epistemic vs. aleatoric uncertainty", "category": "foundations", "short": "Epistemic uncertainty is reducible by gathering data; aleatoric uncertainty is irreducible noise in the environment.", "definition": "A fundamental distinction for curious agents: epistemic (model) uncertainty arises from insufficient data — it can be reduced by exploring. Aleatoric (environment) uncertainty is inherent randomness that cannot be explained by any model — exploring it yields no new knowledge. A good curiosity signal rewards exploration of epistemic uncertainty and ignores aleatoric uncertainty. The noisy-TV problem arises when prediction-error curiosity cannot distinguish the two: a random TV is maximally aleatoric but generates infinite prediction error.", "example": "A flickering TV shows irreducible aleatoric noise — no model will ever predict the next frame. An unseen room is epistemically uncertain — exploring it reduces the agent's model uncertainty.", "related": [ "bayesian-surprise", "prediction-error-curiosity", "noisy-tv-problem", "information-gain" ], "source": "Murphy — Probabilistic Machine Learning (epistemic vs. aleatoric uncertainty); Pathak ICM arXiv:1705.05363 (noisy-TV discussion)" }, { "slug": "experimenter", "term": "Experimenter (WorldLLM)", "category": "the-worldllm-loop", "short": "WorldLLM's curiosity-driven RL agent that seeks transitions the current theory finds surprising.", "definition": "One realization of curiosity-driven / intrinsic-motivation exploration (its general anchor): a policy trained to reach environment transitions to which the Theorist's current natural-language theory assigns LOW log-likelihood (high Bayesian surprise). It generates the data that forces the theory to improve. It is the 'curiosity' half of the WorldLLM loop and an instance of intrinsic-motivation exploration — NOT a general RL control policy and NOT RLHF.", "example": "The Experimenter steers toward a transition the current theory rates implausible; that surprising transition becomes the evidence the Theorist must explain, driving a theory update.", "related": [ "intrinsic-motivation", "bayesian-surprise", "surprising-transition", "theorist", "worldllm" ], "source": "WorldLLM arXiv:2506.06725 (Experimenter); Pathak ICM arXiv:1705.05363 (curiosity-driven exploration anchor); Sutton & Barto (intrinsic motivation)" }, { "slug": "exploration", "term": "Exploration (in RL)", "category": "foundations", "short": "The strategy of visiting unknown or uncertain states to gather information, trading off against exploitation of known rewards.", "definition": "The exploration/exploitation trade-off is a foundational problem in RL: an agent must sometimes choose actions that gather information (exploration) rather than actions that maximize known reward (exploitation). Intrinsic motivation is one family of exploration strategies — rewarding the agent for visiting novel or surprising states. Other families include epsilon-greedy, UCB, and Thompson sampling. Exploration is the general category; curiosity-driven exploration is a specific instance.", "example": "An agent using epsilon-greedy exploration takes random actions with probability ε to discover states it has not visited; an ICM-equipped agent uses prediction error as a directed exploration signal instead.", "related": [ "intrinsic-motivation", "world-model", "model-based-vs-model-free" ], "source": "Sutton & Barto — Reinforcement Learning ch. 2 (exploration/exploitation) and ch. on planning" }, { "slug": "free-energy-minimization", "term": "Free-energy minimization", "category": "methods-frameworks", "short": "The variational objective in active inference: minimizing the gap between the agent's model and the true world.", "definition": "In the free-energy principle, the variational free energy F = KL[Q(world) || P(world | obs)] + H[P(obs | model)] is the objective agents minimize. Minimizing F achieves two things simultaneously: (1) updating the agent's beliefs toward the evidence (perception / Bayesian posterior update), and (2) acting to make the evidence match the prior (goal-seeking). Epistemic free energy (the curiosity component) is the expected F reduction from an action — how much the action would reduce the agent's uncertainty.", "example": "An agent's free energy is high when its belief about its position is uncertain; it takes an action that will yield the most informative observation — minimizing expected free energy by reducing epistemic uncertainty.", "related": [ "active-inference", "bayesian-inference", "information-gain", "active-inference-agent" ], "source": "Friston — Active inference (free-energy principle, DeepMind/UCL); Murphy — Probabilistic Machine Learning (variational inference, KL divergence)" }, { "slug": "hypothesis-refinement", "term": "Hypothesis refinement", "category": "methods-frameworks", "short": "Iteratively revising a hypothesis (theory) to better explain observed evidence, using likelihood scoring.", "definition": "The general process of updating a working hypothesis to increase its likelihood given observed evidence — analogous to Bayesian posterior update but not always fully Bayesian. The WorldLLM Theorist is one realization: it refines a natural-language theory of environment dynamics by scoring candidate theories by their likelihood on collected transitions and proposing revisions via the LLM's in-context reasoning. BoxingGym formalizes this as automated experimental design.", "example": "A scientist refines the hypothesis 'temperature causes plant wilting' to 'high temperature AND low humidity cause plant wilting' after observing that wilting does not occur in high-humidity heat — a hypothesis-refinement step.", "related": [ "bayesian-inference", "proposal-distribution", "theorist", "boxinggym", "information-gain" ], "source": "Murphy — Probabilistic Machine Learning (model selection, Bayesian hypothesis testing); BoxingGym arXiv:2501.01540 (automated hypothesis refinement)" }, { "slug": "jepa", "term": "I-JEPA (Joint-Embedding Predictive Architecture)", "category": "world-models", "short": "A non-generative latent world model that predicts abstract representations of image regions, not pixel-level reconstructions.", "definition": "Assran et al.'s I-JEPA (arXiv:2301.08243): a self-supervised learning architecture that predicts the abstract representation of target image regions from context regions in latent space — a latent predictive world model that does not reconstruct pixels. The key contrast with generative models (VAE/diffusion) is that JEPA predicts in embedding space, avoiding the wasted capacity of pixel-level generation. It enters this World as a world-model predictor (a latent predictive architecture), not as a curiosity or planning system.", "example": "I-JEPA predicts the latent representation of a masked image patch from visible context, learning semantic features without generation — the prediction target is abstract, not pixel-level.", "related": [ "world-model", "dreamer", "worldllm" ], "source": "I-JEPA arXiv:2301.08243; Sutton & Barto (model-based RL / world model)" }, { "slug": "icm", "term": "ICM (Intrinsic Curiosity Module)", "category": "world-models", "short": "A self-supervised curiosity system where intrinsic reward = forward model prediction error in a learned feature space.", "definition": "Pathak et al.'s Intrinsic Curiosity Module (arXiv:1705.05363): a self-supervised system that learns a feature space of states via an inverse model (predicting actions from state transitions) and trains a forward model in that space. Intrinsic reward = the forward model's prediction error. ICM is the canonical instance of prediction-error curiosity and the system that introduced the noisy-TV problem as a named failure mode. It contrasts with Bayesian-surprise-based systems: ICM rewards raw prediction error; Bayesian surprise rewards posterior change.", "example": "An ICM-equipped agent explores a Vizdoom maze efficiently because prediction error is high in unexplored rooms and low in familiar corridors — directing attention toward novelty without an extrinsic reward.", "related": [ "prediction-error-curiosity", "noisy-tv-problem", "intrinsic-motivation", "world-model", "bayesian-surprise" ], "source": "Pathak et al., ICM, arXiv:1705.05363; Sutton & Barto (intrinsic motivation)" }, { "slug": "in-context-learning", "term": "In-context learning", "category": "foundations", "short": "An LLM's ability to update its behavior based on examples or evidence provided in the prompt, without weight updates.", "definition": "The capacity of large language models to adapt their outputs to new evidence or examples provided in the context window, without any gradient-based parameter update. In the WorldLLM Theorist, in-context learning is the substrate for hypothesis refinement: the Theorist updates its theory by conditioning on new transitions in the context, not by fine-tuning. This makes the Theorist's 'Bayesian update' an in-context, not a gradient-based, operation.", "example": "The Theorist is shown five new surprising transitions in its context window; its next theory proposal conditions on all five, refining the hypothesis without any model weight changes.", "related": [ "hypothesis-refinement", "proposal-distribution", "theorist" ], "source": "Sutton & Barto (model-based update analog); Murphy — Probabilistic Machine Learning (Bayesian in-context update framing)" }, { "slug": "information-gain", "term": "Information gain", "category": "curiosity-signals", "short": "The expected reduction in entropy of the agent's belief over hypotheses, given an action/observation.", "definition": "The expected KL divergence between the posterior and prior over world models or hypotheses after taking an action and observing the outcome. Information gain formalizes 'how much will I learn from this action?' and is the objective behind optimal Bayesian experimental design. It is a generalization of Bayesian surprise to expectations over possible outcomes. Active-inference agents maximize expected information gain (epistemic value) as part of their free-energy objective.", "example": "Between two experiments, an agent chooses the one with higher expected information gain — the one whose outcome will most narrow the set of plausible theories about the environment.", "related": [ "bayesian-surprise", "bayesian-inference", "active-inference", "epistemic-vs-aleatoric", "boxinggym" ], "source": "Murphy — Probabilistic Machine Learning (information gain, Bayesian experimental design); Friston active inference (epistemic value)" }, { "slug": "intrinsic-motivation", "term": "Intrinsic motivation", "category": "foundations", "short": "Reward an agent generates internally to drive exploration, absent external task reward.", "definition": "A self-generated drive to explore — the agent rewards itself for novelty, surprise, or learning progress rather than for an extrinsic task signal. It is the umbrella concept; prediction-error curiosity and Bayesian surprise are two distinct *instances* of it (a distinction this World exists to make clear). Sutton & Barto treat it as exploration shaping; it is NOT a control algorithm and NOT RLHF.", "example": "An agent with no game score still explores a new room because the room is unpredictable — its intrinsic reward is high where its world model is wrong.", "related": [ "prediction-error-curiosity", "bayesian-surprise", "exploration", "world-model" ], "source": "Sutton & Barto — Reinforcement Learning ch. on exploration/intrinsic motivation; Pathak ICM arXiv:1705.05363 (intro)" }, { "slug": "intrinsic-reward-hacking", "term": "Intrinsic reward hacking", "category": "failure-modes", "short": "An agent exploits the curiosity signal itself as a reward to hack rather than as a guide to genuine exploration.", "definition": "The general failure mode where an agent finds policies that maximize the intrinsic curiosity reward without achieving the intended effect (exploring novel, informative states). This includes the noisy-TV problem (aleatoric trap), detachment (feature-space exploitation), and count-based hacking (cycling through rarely-visited but uninformative states). It is the intrinsic-reward analog of extrinsic reward hacking. The fix is a curiosity signal that rewards epistemic value, not raw prediction error or visit rarity.", "example": "A count-based exploration agent repeatedly visits a random-noise tile because it is never fully familiar — maximizing novelty without discovering anything useful.", "related": [ "detachment", "noisy-tv-problem", "prediction-error-curiosity", "bayesian-surprise", "information-gain" ], "source": "Pathak ICM arXiv:1705.05363; Sutton & Barto (exploration, reward shaping); Murphy (epistemic value)" }, { "slug": "log-likelihood-surprise", "term": "Log-likelihood surprise", "category": "curiosity-signals", "short": "Surprise measured as the negative log-probability a model assigns to an observed outcome: -log P(observation | model).", "definition": "The basic information-theoretic definition of surprise: -log P(x | model). High log-likelihood surprise means the current model finds the observation improbable. WorldLLM's Experimenter uses this as its curiosity signal — it seeks transitions the Theorist's current NL theory assigns low log-likelihood (high log-likelihood surprise). It is a special case of Bayesian surprise when the prior and posterior are collapsed to the current model's point estimate.", "example": "The Theorist's theory assigns probability 0.02 to the observed transition; log-likelihood surprise = -log(0.02) ≈ 3.9 nats — high, making this a target for the Experimenter.", "related": [ "bayesian-surprise", "surprising-transition", "information-gain", "prediction-error-curiosity" ], "source": "Murphy — Probabilistic Machine Learning (log-likelihood, surprise); WorldLLM arXiv:2506.06725 (log-likelihood as the Experimenter's curiosity signal)" }, { "slug": "model-based-vs-model-free", "term": "Model-based vs. model-free RL", "category": "foundations", "short": "Model-based agents plan using an explicit world model; model-free agents learn a policy or value function directly from experience.", "definition": "The primary architectural split in RL: model-free agents (Q-learning, policy gradients) learn to act by directly optimizing from environment rewards without an explicit environment model — they require many real-environment samples. Model-based agents (Dyna, Dreamer, WorldLLM) maintain an explicit world model and can plan, simulate, or reason about actions before executing them — more sample-efficient but require a world model that is accurate enough. Curiosity systems can be attached to either, but model-based agents have more natural access to the transition model that curiosity signals need.", "example": "A model-free PPO agent requires millions of game frames to learn Atari; DreamerV3 (model-based) achieves similar performance with far fewer real-environment steps by training its policy on imagined rollouts.", "related": [ "world-model", "transition-model", "dreamer", "exploration" ], "source": "Sutton & Barto — Reinforcement Learning ch. 8 (model-based RL, Dyna)" }, { "slug": "noisy-tv-problem", "term": "Noisy-TV problem", "category": "failure-modes", "short": "Prediction-error curiosity gets trapped by inherently unpredictable (aleatoric) stimuli.", "definition": "A failure mode of prediction-error curiosity: an agent rewarded for prediction error fixates on stochastic, unlearnable stimuli (a TV showing static) because prediction error never falls. It is why distinguishing EPISTEMIC uncertainty (reducible — worth exploring) from ALEATORIC uncertainty (irreducible noise — a trap) matters, and a key argument for Bayesian-surprise / information-gain signals that reward *model improvement* rather than raw unpredictability.", "example": "Curiosity by prediction error stalls forever in front of a random-noise generator; a Bayesian-surprise agent leaves once its posterior stops moving.", "related": [ "prediction-error-curiosity", "bayesian-surprise", "information-gain", "epistemic-vs-aleatoric" ], "source": "Pathak ICM arXiv:1705.05363 (noisy-TV discussion); Latent Bayesian Surprise arXiv:2104.07495; Murphy (epistemic vs. aleatoric)" }, { "slug": "novelty-signal", "term": "Novelty signal", "category": "curiosity-signals", "short": "An intrinsic reward proportional to how rarely a state or observation has been visited.", "definition": "A count-based or density-model intrinsic reward: the agent is rewarded for visiting states or observations it has encountered rarely (low visit count or low density under a learned model). Novelty signals are the simplest instance of intrinsic motivation and predate ICM. They do not require a forward model or posterior over theories — only a count or density estimate. They suffer from the same noisy-TV vulnerability if random states have low visit count but are uninformative.", "example": "An agent with a visit-count novelty signal explores a grid world systematically, but gets stuck visiting rare but uninformative cells (e.g. random-noise tiles) that never become familiar.", "related": [ "intrinsic-motivation", "prediction-error-curiosity", "bayesian-surprise", "exploration" ], "source": "Sutton & Barto — Reinforcement Learning (exploration, optimistic initialization); Murphy (density estimation)" }, { "slug": "pomdp", "term": "POMDP (Partially Observable Markov Decision Process)", "category": "foundations", "short": "An MDP where the agent does not directly observe the full environment state — it infers a belief state.", "definition": "A Partially Observable Markov Decision Process extends the standard MDP with a hidden state space: the agent receives observations rather than full state information and must maintain a belief distribution (a posterior over possible states). World models are especially valuable in POMDPs because the model can help the agent infer the latent state from partial observations. Curiosity signals in POMDPs reward exploration of regions of state-space where the belief distribution is uncertain.", "example": "A robot navigating a building does not see around corners; it maintains a belief over its location and uses its world model to predict what it will observe when it rounds a corner.", "related": [ "world-model", "transition-model", "model-based-vs-model-free", "epistemic-vs-aleatoric" ], "source": "Sutton & Barto — Reinforcement Learning (POMDPs, belief states)" }, { "slug": "prediction-error-curiosity", "term": "Prediction-error curiosity (ICM)", "category": "curiosity-signals", "short": "Intrinsic reward = how wrong the agent's forward model is about the next state.", "definition": "One *instance* of intrinsic motivation (Pathak's ICM): the agent's intrinsic reward is the error of a learned forward dynamics model in a self-supervised feature space. High prediction error = high curiosity. It CONTRASTS WITH Bayesian surprise: prediction-error rewards being wrong about the next state; Bayesian surprise rewards the *change in the posterior over models*. Conflating the two is the exact confusion this term resolves. Its known pathology is the noisy-TV problem.", "example": "An agent is drawn to a flickering screen because its forward model never predicts the next frame — prediction error stays high forever (the noisy-TV trap).", "related": [ "intrinsic-motivation", "bayesian-surprise", "noisy-tv-problem", "world-model", "icm", "epistemic-vs-aleatoric" ], "source": "Pathak et al., ICM, arXiv:1705.05363; Sutton & Barto (intrinsic reward)" }, { "slug": "proposal-distribution", "term": "Proposal distribution", "category": "methods-frameworks", "short": "A distribution from which candidate hypotheses or actions are sampled, then accepted/rejected or scored.", "definition": "In Bayesian inference and MCMC, the proposal distribution Q(H' | H) generates candidate hypotheses H' given the current hypothesis H; a scoring or acceptance step decides whether to adopt H'. The WorldLLM Theorist uses the LLM itself as the proposal distribution over NL theory revisions — the LLM proposes candidate theories conditioned on collected observations and the current theory, and the theory with highest transition likelihood is adopted. This is the technical sense in which the Theorist is 'Bayesian hypothesis refinement with an LLM as the proposal distribution.'", "example": "In Metropolis-Hastings, the proposal distribution generates a candidate parameter value; in WorldLLM, the LLM generates a candidate theory revision. In both cases, the proposal is scored by evidence before acceptance.", "related": [ "bayesian-inference", "hypothesis-refinement", "theorist" ], "source": "Murphy — Probabilistic Machine Learning ch. 24 (MCMC, proposal distributions); WorldLLM arXiv:2506.06725 (Theorist as proposal distribution)" }, { "slug": "provenance-gate-anchor", "term": "Provenance gate as reality-anchor", "category": "methods-frameworks", "short": "[BUILT] QuKaiZen's assertClosedSourcedGraph gate: the mechanism that keeps autonomous hypotheses RAW until sourced evidence supports them.", "definition": "[BUILT] The assertClosedSourcedGraph function (src/gate.ts, QuKaiZen DaC engine) is the provenance gate: it enforces three laws — (1) every term carries a non-empty source, (2) every category is declared in the spec, (3) every related edge resolves to a compiled term. In the context of a WorldLLM-style Theorist/Experimenter loop, the gate is the reality-anchor: a Theorist hypothesis stays RAW (uncompiled) until the gate confirms it is sourced to verifiable evidence. This is BUILT today; the autonomous loop running on it is ROADMAP (see reality-anchored-theorist).", "example": "[BUILT] When assemble-world.mts runs, the gate refuses to compile any term whose source field is empty — keeping unsourced hypotheses out of the COMPILED truth corpus. This is the same mechanism that would keep a Theorist's unverified hypothesis RAW in a live loop.", "related": [ "hypothesis-refinement", "ungrounded-hypothesis-drift", "reality-anchored-theorist", "theorist" ], "source": "WorldLLM arXiv:2506.06725 (hypothesis RAW-until-grounded framing); Murphy — Probabilistic Machine Learning (grounded inference, sourced evidence); QuKaiZen charter — DAC_ENGINE.md (assertClosedSourcedGraph), CLAUDE.md (provenance gate as autonomy safety mechanism)" }, { "slug": "reality-anchored-theorist", "term": "Reality-anchored Theorist loop (QuKaiZen)", "category": "the-worldllm-loop", "short": "[ROADMAP] QuKaiZen's intended Theorist/Experimenter loop, safety-anchored by the provenance gate.", "definition": "[ROADMAP] The QuKaiZen-internal application of the WorldLLM Theorist/Experimenter loop (arXiv:2506.06725): an autonomous curiosity-driven loop in which hypotheses generated by the Theorist remain RAW until the provenance gate (assertClosedSourcedGraph — BUILT today in src/gate.ts) confirms sourced evidence supports them. This loop does not yet run end-to-end; it is the autonomy ROADMAP. The gate (BUILT) is the reality-anchor; the loop running on it is ROADMAP. See provenance-gate-anchor for the BUILT component.", "example": "[ROADMAP] When built, the loop will propose a hypothesis about a plant's condition, seek surprising observations (Experimenter), and only promote the refined hypothesis to COMPILED status once the gate confirms sourced evidence — the safety property the provenance gate provides.", "related": [ "theorist", "worldllm", "provenance-gate-anchor", "ungrounded-hypothesis-drift", "experimenter" ], "source": "WorldLLM arXiv:2506.06725 (Theorist/Experimenter loop); Sutton & Barto (model-based RL, autonomy loop anchor); QuKaiZen charter — CLAUDE.md (autonomy ROADMAP), DAC_ENGINE.md (provenance gate)" }, { "slug": "surprising-transition", "term": "Surprising (low-log-likelihood) transition", "category": "the-worldllm-loop", "short": "An observed transition the current theory rates as improbable — the WorldLLM curiosity signal.", "definition": "The concrete signal the Experimenter maximizes: a state→action→next-state transition assigned low log-likelihood under the Theorist's current theory. It is WorldLLM's operationalization of Bayesian surprise as an LLM log-likelihood, and the bridge from the curiosity signal to the theory update. It contrasts with raw prediction-error curiosity (ICM): both seek 'surprising' transitions, but log-likelihood surprise rewards model-inconsistency in natural language, while ICM rewards forward-model error in feature space.", "example": "The theory says 'pushing the box does nothing'; the agent finds the box moves — a low-log-likelihood transition that triggers a theory revision.", "related": [ "bayesian-surprise", "log-likelihood-surprise", "experimenter", "theorist", "updated-theory", "prediction-error-curiosity" ], "source": "WorldLLM arXiv:2506.06725 (surprise as transition log-likelihood); Murphy (surprise as -log p); Latent Bayesian Surprise arXiv:2104.07495" }, { "slug": "theorist", "term": "Theorist (WorldLLM)", "category": "the-worldllm-loop", "short": "WorldLLM's 2nd LLM that refines a natural-language theory of environment dynamics via Bayesian updating.", "definition": "One realization of Bayesian hypothesis refinement with an LLM as the proposal distribution (its general anchor). The Theorist proposes/revises a natural-language theory of the environment's dynamics, scored by how well it predicts collected transitions (likelihood); surprising transitions from the Experimenter drive the update. It is the 'world-model' half of the loop — the world model here is an NL theory, NOT a JEPA latent predictor (a distinction this World makes explicit). The Theorist does NOT execute actions; the Experimenter does.", "example": "After seeing the box move, the Theorist proposes a revised theory 'heavy objects move when pushed', which raises the likelihood of the new transitions and lowers the Experimenter's next target.", "related": [ "hypothesis-refinement", "proposal-distribution", "bayesian-inference", "experimenter", "updated-theory", "worldllm", "world-model" ], "source": "WorldLLM arXiv:2506.06725 (Theorist as proposal distribution); Murphy (Bayesian posterior update); BoxingGym arXiv:2501.01540 (automated hypothesis refinement anchor)" }, { "slug": "transition-model", "term": "Transition model", "category": "foundations", "short": "The component of a world model that predicts the next state given current state and action: P(s' | s, a).", "definition": "The transition model (or dynamics model) specifies P(s' | s, a) — the probability distribution over next states given the current state and action. It is the core of any world model. Model-based RL agents learn a transition model from data and use it for planning or curiosity. ICM's forward model is a learned transition model in feature space; DreamerV3's RSSM is a learned stochastic transition model in latent space.", "example": "A robot's transition model predicts that pushing a box left moves it left with 90% probability and leaves it stationary with 10% — enabling the robot to plan whether pushing will clear a path.", "related": [ "world-model", "model-based-vs-model-free", "dreamer", "icm" ], "source": "Sutton & Barto — Reinforcement Learning (model-based RL, Dyna); Ha & Schmidhuber arXiv:1803.10122" }, { "slug": "ungrounded-hypothesis-drift", "term": "Ungrounded-hypothesis drift", "category": "failure-modes", "short": "A Theorist-style agent's theory drifts away from reality because unverified hypotheses are promoted to truth without sourced evidence.", "definition": "[ROADMAP] A failure mode specific to Theorist/Experimenter-style loops: if the Theorist's proposed hypotheses are promoted to the agent's 'compiled' world model without requiring sourced, evidence-backed verification, the theory drifts from reality — the agent acts on a model that was never grounded. In WorldLLM this is prevented by the scoring mechanism (only theories that predict transitions well survive). In QuKaiZen's provenance gate (BUILT — assertClosedSourcedGraph), it is prevented at compile time by requiring every term to have a non-empty, traceable source. The QuKaiZen loop (ROADMAP) will need this gate as its reality-anchor; the gate itself is BUILT today.", "example": "A Theorist loop without a provenance gate promotes 'the environment's physics are non-deterministic' after a single noisy observation, and the Experimenter optimizes for a world that doesn't exist — the loop hallucinates.", "related": [ "provenance-gate-anchor", "theorist", "reality-anchored-theorist", "hypothesis-refinement", "bayesian-surprise" ], "source": "WorldLLM arXiv:2506.06725 (grounding via transition likelihood); Murphy — Probabilistic Machine Learning (model grounding, evidence-based inference); QuKaiZen charter — CLAUDE.md (provenance gate as reality-anchor), DAC_ENGINE.md" }, { "slug": "updated-theory", "term": "Updated theory of environment dynamics", "category": "the-worldllm-loop", "short": "The refined NL theory the loop converges toward — WorldLLM's improving world model.", "definition": "The output of one WorldLLM loop iteration: a revised natural-language theory with higher likelihood on observed transitions, which then re-defines what the Experimenter finds surprising. Iterating closes the loop (propose → seek surprise → refine). It is the 'updated world model' in NL form. It contrasts with a latent world model (JEPA/Dreamer): those encode dynamics as learned embeddings; the updated theory encodes them as natural language propositions that a human could read.", "example": "Across iterations the theory grows from 'objects are static' to a near-complete account of the environment's dynamics; surprising transitions become rare as it converges.", "related": [ "theorist", "experimenter", "surprising-transition", "hypothesis-refinement", "world-model", "worldllm" ], "source": "WorldLLM arXiv:2506.06725 (theory refinement loop); Sutton & Barto (model-based learning / world model)" }, { "slug": "world-model", "term": "World model", "category": "foundations", "short": "An internal model of environment dynamics that an agent uses to predict outcomes of actions.", "definition": "A learned or hand-coded model of an environment's transition dynamics — how states evolve under actions. World models enable model-based planning (imagined rollouts) without real environment interaction. The concept is general; different systems instantiate it differently: DreamerV3 uses a latent RSSM; I-JEPA uses a latent predictive embedding; WorldLLM uses a natural-language theory. This disambiguation is the core value of this World for the applied AI builder.", "example": "A chess-playing agent with a world model simulates future board states without making real moves; a robot with a world model plans a path without physical trial-and-error.", "related": [ "transition-model", "model-based-vs-model-free", "dreamer", "jepa", "worldllm", "intrinsic-motivation" ], "source": "Sutton & Barto — Reinforcement Learning (model-based vs. model-free, Dyna architecture); Ha & Schmidhuber World Models arXiv:1803.10122" }, { "slug": "worldllm", "term": "WorldLLM", "category": "world-models", "short": "A curiosity-driven world-modeling system pairing an Experimenter (RL) with a Theorist (LLM) that refines a natural-language theory of environment dynamics.", "definition": "A world-modeling system (arXiv:2506.06725) in which two LLMs collaborate: an Experimenter (curiosity-driven RL policy) that seeks low-log-likelihood transitions, and a Theorist that refines a natural-language theory of the environment's dynamics as a Bayesian hypothesis-refinement process. The world model here is a natural-language theory, not a latent embedding — which distinguishes it from Dreamer (latent) and JEPA (latent predictive). The system operationalizes Bayesian surprise as LLM log-likelihood and hypothesis refinement as in-context Bayesian update.", "example": "WorldLLM applied to the BabyAI environment converges from a null theory to a near-complete description of grid-world dynamics through iterated curiosity-driven experiments and Theorist revisions.", "related": [ "experimenter", "theorist", "surprising-transition", "updated-theory", "intrinsic-motivation", "bayesian-surprise", "world-model" ], "source": "WorldLLM arXiv:2506.06725; Sutton & Barto (intrinsic motivation, model-based RL anchor); Murphy (Bayesian inference anchor)" } ] }