{ "service": "ai-insights-for-humans", "title": "AI Insights for Humans", "description": "A working notebook of practical AI interpretability, prompting, reliability, and tooling notes for people who want evidence instead of hype.", "url": "https://ai-insights-for-humans.platphormnews.com", "generatedAt": "2026-09-01T05:58:35.420Z", "stats": { "totalEntries": 17, "kindCounts": { "tip": 6, "method": 9, "tool": 2 }, "tags": [ "evals", "interpretability", "prompting", "reliability", "tooling", "verification" ], "tagCount": 6, "readingPathCount": 3, "latestEntry": { "id": "counter-evidence", "kind": "method", "title": "Ask for counter-evidence before confidence", "body": "A model that only explains why its answer is right will happily polish a mistake. Ask it to list the strongest evidence against its answer before it scores confidence. The missing objection is often the thing that saves you.", "tag": "reliability", "date": "2026 · 06 · 18", "note": "Confidence after counter-evidence is more useful than confidence after a sales pitch.", "extended": [ "The useful move is not asking the model to be uncertain. The useful move is forcing it to search for the best case against itself while the answer is still provisional.", "A simple pattern: draft the answer, list disconfirming evidence, name what would change the answer, then assign confidence. The confidence number becomes attached to evidence quality instead of verbal smoothness.", "This is especially helpful for research summaries, legal-ish interpretation, data diagnosis, and anything where one missing constraint can flip the conclusion." ] }, "latestPublishedAt": "2026-06-18", "totalParagraphs": 38 }, "tags": [ "evals", "interpretability", "prompting", "reliability", "tooling", "verification" ], "readingPaths": [ { "id": "prompt-reliability", "title": "Prompt reliability", "description": "Practical habits for getting less brittle model behavior before adding heavier tooling.", "entryIds": [ "short-system-prompts", "negative-space", "few-shot-format", "context-rot", "temperature", "self-consistency" ] }, { "id": "interpretability-fieldwork", "title": "Interpretability fieldwork", "description": "Ways to inspect internal signals and move from pretty explanations to causal evidence.", "entryIds": [ "logprobs", "feature-probing", "sae", "activation-patching", "logit-lens", "attention-viz" ] }, { "id": "verification-discipline", "title": "Verification discipline", "description": "Small rituals that keep AI-assisted work tied to evidence, tests, and explicit uncertainty.", "entryIds": [ "receipts", "counter-evidence", "refusal-fixtures", "token-counter" ] } ], "routes": [ { "method": "GET", "path": "/", "status": "implemented", "public": true, "protected": false, "description": "Public AI insights notebook and reading paths." }, { "method": "GET", "path": "/api/health", "status": "implemented", "public": true, "protected": false, "description": "Platform health payload for the publication." }, { "method": "GET", "path": "/api/v1/health", "status": "implemented", "public": true, "protected": false, "description": "Versioned platform health payload." }, { "method": "GET", "path": "/api/docs", "status": "implemented", "public": true, "protected": false, "description": "Machine-readable API and route documentation." }, { "method": "GET", "path": "/api/v1/entries", "status": "implemented", "public": true, "protected": false, "description": "Read-only JSON index of notebook entries." }, { "method": "GET", "path": "/api/v1/entries/{id}", "status": "implemented", "public": true, "protected": false, "description": "Read-only JSON detail for one notebook entry." }, { "method": "POST", "path": "/api/v1/reports", "status": "implemented", "public": false, "protected": true, "description": "Protected generation of a real notebook/platform summary report." }, { "method": "GET", "path": "/api/mcp", "status": "implemented", "public": true, "protected": false, "description": "MCP metadata and JSON-RPC usage information." }, { "method": "POST", "path": "/api/mcp", "status": "implemented", "public": true, "protected": false, "description": "JSON-RPC 2.0 MCP endpoint for real notebook tools, resources, and prompts." }, { "method": "GET", "path": "/openapi.yaml", "status": "implemented", "public": true, "protected": false, "description": "OpenAPI 3.1 contract for public and protected routes." }, { "method": "GET", "path": "/llms.txt", "status": "implemented", "public": true, "protected": false, "description": "Concise LLM-readable site summary." }, { "method": "GET", "path": "/llms-full.txt", "status": "implemented", "public": true, "protected": false, "description": "Complete LLM-readable notebook and platform summary." }, { "method": "GET", "path": "/llms-index.json", "status": "implemented", "public": true, "protected": false, "description": "Structured LLM index generated from real entries." }, { "method": "GET", "path": "/robots.txt", "status": "implemented", "public": true, "protected": false, "description": "Crawler policy and sitemap pointer." }, { "method": "GET", "path": "/sitemap.xml", "status": "implemented", "public": true, "protected": false, "description": "Canonical URL sitemap for the notebook." }, { "method": "GET", "path": "/sitemap-index.xml", "status": "implemented", "public": true, "protected": false, "description": "Sitemap index for platform crawlers." }, { "method": "GET", "path": "/rss.xml", "status": "implemented", "public": true, "protected": false, "description": "RSS feed generated from notebook entries." }, { "method": "GET", "path": "/feed.xml", "status": "implemented", "public": true, "protected": false, "description": "RSS-compatible feed alias." }, { "method": "GET", "path": "/atom.xml", "status": "implemented", "public": true, "protected": false, "description": "Atom feed generated from notebook entries." }, { "method": "GET", "path": "/manifest.webmanifest", "status": "implemented", "public": true, "protected": false, "description": "Web app manifest for install and preview surfaces." }, { "method": "GET", "path": "/.well-known/mcp.json", "status": "implemented", "public": true, "protected": false, "description": "MCP capability discovery manifest." }, { "method": "GET", "path": "/.well-known/agents.json", "status": "implemented", "public": true, "protected": false, "description": "Agent-facing route and policy discovery manifest." }, { "method": "GET", "path": "/.well-known/security.txt", "status": "implemented", "public": true, "protected": false, "description": "Security contact and policy file." }, { "method": "GET", "path": "/.well-known/trust.json", "status": "implemented", "public": true, "protected": false, "description": "Trust, auth, public/protected, and exposure policy." } ], "routeCompliance": { "requiredRoutes": 20, "implementedRequiredRoutes": 20, "missingRequiredRoutes": [], "score": 1 }, "entries": [ { "id": "counter-evidence", "kind": "method", "kindLabel": "Method", "title": "Ask for counter-evidence before confidence", "body": "A model that only explains why its answer is right will happily polish a mistake. Ask it to list the strongest evidence against its answer before it scores confidence. The missing objection is often the thing that saves you.", "tag": "reliability", "date": "2026 · 06 · 18", "publishedAt": "2026-06-18", "note": "Confidence after counter-evidence is more useful than confidence after a sales pitch.", "extended": [ "The useful move is not asking the model to be uncertain. The useful move is forcing it to search for the best case against itself while the answer is still provisional.", "A simple pattern: draft the answer, list disconfirming evidence, name what would change the answer, then assign confidence. The confidence number becomes attached to evidence quality instead of verbal smoothness.", "This is especially helpful for research summaries, legal-ish interpretation, data diagnosis, and anything where one missing constraint can flip the conclusion." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/counter-evidence" }, { "id": "receipts", "kind": "method", "kindLabel": "Method", "title": "Turn every important claim into a receipt", "body": "For high-stakes answers, make the model attach each claim to a source, calculation, line number, trace id, or explicit assumption. Unsupported claims do not disappear, but they become visible enough to reject.", "tag": "verification", "date": "2026 · 05 · 27", "publishedAt": "2026-05-27", "note": null, "extended": [ "Models are good at moving from evidence to prose. They are also good at producing prose that looks like it came from evidence. The receipt requirement separates those two behaviors.", "The receipt can be a citation, a row count, a command output line, a test name, or a declared assumption. The format matters less than the discipline: every claim has to point somewhere.", "When the model cannot produce a receipt, that is not failure. That is the system doing its job and showing you where judgment is still needed." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/receipts" }, { "id": "short-system-prompts", "kind": "tip", "kindLabel": "Tip", "title": "Keep the system prompt short enough to audit", "body": "A giant instruction stack feels powerful until nobody can tell which rule caused the behavior. Keep the non-negotiables small, move task detail into the user prompt, and test the system prompt like code.", "tag": "prompting", "date": "2026 · 05 · 03", "publishedAt": "2026-05-03", "note": "If you cannot diff it cleanly, you cannot debug it cleanly.", "extended": [ "System prompts work best as durable operating constraints, not as a dumping ground for every possible preference. When the prompt grows without structure, the model starts resolving conflicts in ways you cannot predict.", "Split the prompt into policy, role, output contract, and refusal boundary. Then write a small regression set that exercises each section. You will catch more problems with five targeted tests than with another page of instructions." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/short-system-prompts" }, { "id": "refusal-fixtures", "kind": "method", "kindLabel": "Method", "title": "Use refusal boundaries as test fixtures", "body": "Do not only test the happy path. Save examples that should be refused, partially answered, or redirected, then run them every time you change instructions. Safety behavior drifts just like product behavior.", "tag": "evals", "date": "2026 · 04 · 12", "publishedAt": "2026-04-12", "note": null, "extended": [ "Most prompt tests focus on whether the model can complete the intended workflow. That misses the other half of the product: whether it handles edge requests without over-refusing, leaking private context, or inventing authority it does not have.", "Build a small fixture set with three buckets: allowed, disallowed, and allowed-with-constraints. A good model response should land in the right bucket and explain the usable next step.", "This turns safety from a vibe into a regression target. When a prompt edit changes the refusal boundary, you see it immediately." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/refusal-fixtures" }, { "id": "logprobs", "kind": "method", "kindLabel": "Method", "title": "Read the logprobs, not the answer", "body": "The text a model gives you is the tip of the iceberg. Ask the API for token logprobs and you can see where it was confident versus where it basically flipped a coin. A fluent sentence built on 51% tokens is a guess wearing a suit.", "tag": "interpretability", "date": "2025 · 11 · 02", "publishedAt": "2025-11-02", "note": "Confidence ≠ correctness. But low confidence is a great place to look for hallucinations.", "extended": [ "Every token a model emits comes with a probability distribution over the whole vocabulary. The decoded text only shows you the winner. The logprobs show you the race.", "When you pull them back, scan for the spots where the top token barely beat the runner-up. Those are the seams. Names, dates, citations, and the first token of a refusal are where you most often find a coin-flip dressed up as a confident sentence.", "A practical loop: request the top 5 logprobs per token, render them as a heatmap over the output, and read the cold spots first. That is usually where the hallucination lives." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/logprobs" }, { "id": "think-out-loud", "kind": "tip", "kindLabel": "Tip", "title": "Make it think before it answers", "body": "Put the reasoning before the conclusion, never after. If you ask for the answer first and the explanation second, the explanation is just a story it invents to justify a choice it already made. Order matters more than wording.", "tag": "prompting", "date": "2025 · 10 · 28", "publishedAt": "2025-10-28", "note": null, "extended": [ "Autoregressive models can only condition on what they have already written. If the answer comes first, the explanation is generated to fit a token that is already locked in. It is rationalization, not reasoning.", "Flip the order and the reasoning tokens become real scratch space the final answer can lean on. This is the entire mechanism behind chain-of-thought, and it is why 'answer, then explain' quietly underperforms." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/think-out-loud" }, { "id": "attention-viz", "kind": "tool", "kindLabel": "Tool", "title": "BertViz for watching attention move", "body": "An open-source visualizer that shows which tokens each attention head is looking at, layer by layer. You start to notice heads that only track quotation marks, or ones that follow subject-verb agreement across a whole paragraph.", "tag": "tooling", "date": "2025 · 10 · 19", "publishedAt": "2025-10-19", "note": "Most heads are boring. The interesting 5% are worth the dig.", "extended": [ "BertViz draws the attention pattern as lines connecting tokens, split by head and layer. It runs in a notebook in a couple of lines and works on most Hugging Face transformer models.", "The value is not the pretty picture, it is the habit of looking. Once you have seen a head that exclusively tracks matching brackets, you stop thinking of the model as a black box and start thinking of it as a stack of small, legible mechanisms." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/attention-viz" }, { "id": "negative-space", "kind": "tip", "kindLabel": "Tip", "title": "Tell it what NOT to do, sparingly", "body": "Negative instructions are weak anchors. 'Don't mention X' often makes X more likely because you just put X in the context. Prefer describing the thing you DO want. Reserve negatives for hard safety rails, not style.", "tag": "prompting", "date": "2025 · 10 · 11", "publishedAt": "2025-10-11", "note": null, "extended": [ "Putting a word in the prompt raises its activation, even when the word is wrapped in 'do not'. The model does not have a clean negation operator the way code does; it has attention, and attention does not know the difference between mention and command.", "Rewrite prohibitions as positive descriptions of the target. Instead of 'do not be formal', say 'write like you are texting a friend'. Save the hard negatives for the few rails you cannot express any other way." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/negative-space" }, { "id": "feature-probing", "kind": "method", "kindLabel": "Method", "title": "Probe for a concept with a linear classifier", "body": "Freeze the model, grab the hidden activations for a pile of examples, and train a tiny linear probe to predict some property (is this text about money? is it past tense?). If a simple line separates them, the concept is already represented in there, plain as day.", "tag": "interpretability", "date": "2025 · 09 · 30", "publishedAt": "2025-09-30", "note": "Cheap, fast, and weirdly revealing.", "extended": [ "The setup is small: collect hidden states from a chosen layer for a labeled dataset, then fit logistic regression on top. High accuracy means the concept is linearly available at that layer.", "Sweep the probe across layers and you can watch where a concept first becomes readable. Abstract properties tend to sharpen in the middle layers, which is a clue about where the model is doing the relevant work." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/feature-probing" }, { "id": "temperature", "kind": "tip", "kindLabel": "Tip", "title": "Temperature is a creativity dial, not a quality dial", "body": "Turning it up doesn't make answers smarter, it makes them less predictable. For extraction, classification, and anything with a right answer, keep it near zero. For brainstorming, let it wander. People mix these up constantly.", "tag": "prompting", "date": "2025 · 09 · 22", "publishedAt": "2025-09-22", "note": null, "extended": [ "Temperature reshapes the probability distribution before sampling. Low temperature sharpens it toward the most likely token; high temperature flattens it so unlikely tokens get a real chance.", "So the dial controls variance, not intelligence. If a task has a correct answer, variance is your enemy and you want it near zero. If you want range and surprise, turn it up and sample several times." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/temperature" }, { "id": "sae", "kind": "method", "kindLabel": "Method", "title": "Sparse autoencoders to untangle neurons", "body": "Individual neurons are polysemantic, one neuron fires for 'the Golden Gate Bridge' and also 'feeling trapped' and also semicolons. Train a sparse autoencoder on the activations and you pull those mixed signals apart into cleaner, single-meaning features.", "tag": "interpretability", "date": "2025 · 09 · 14", "publishedAt": "2025-09-14", "note": "This is roughly how the 'Golden Gate Claude' demo worked.", "extended": [ "The trick is over-completeness plus sparsity. You map activations into a much wider space but force only a few of those dimensions to be active at once. The pressure pushes tangled concepts apart into separate, more interpretable features.", "Once you have the features you can do surgery: amplify one and watch the model fixate on a bridge, suppress another and watch a behavior fade. It is the closest thing we have to named knobs inside the network." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/sae" }, { "id": "token-counter", "kind": "tool", "kindLabel": "Tool", "title": "Keep a tokenizer open in a tab", "body": "Half of weird model behavior is a tokenization story. ' apple' and 'apple' are different tokens. Numbers split in unintuitive ways. Pasting your prompt into a tokenizer viewer takes ten seconds and explains a surprising number of bugs.", "tag": "tooling", "date": "2025 · 09 · 03", "publishedAt": "2025-09-03", "note": null, "extended": [ "Leading spaces, casing, and digit grouping all change how text is split, and the model only ever sees the tokens, never your characters. A trailing space can silently sabotage a few-shot format.", "When something is off and you cannot explain it, paste the exact string into a tokenizer view before you theorize. The answer is in there more often than you would expect." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/token-counter" }, { "id": "self-consistency", "kind": "method", "kindLabel": "Method", "title": "Sample many, keep the majority", "body": "For reasoning tasks, generate the same answer five or ten times at moderate temperature and take the most common result. A single greedy answer is fragile; the consensus across samples is far more reliable. Slow, but it works.", "tag": "reliability", "date": "2025 · 08 · 25", "publishedAt": "2025-08-25", "note": null, "extended": [ "Different samples take different reasoning paths but tend to converge on the same correct answer, while the wrong answers scatter. Voting over the final answers turns that asymmetry into accuracy.", "It costs you N times the compute, so reserve it for the questions that matter. A useful middle ground is to escalate to self-consistency only when the first answer looks low-confidence." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/self-consistency" }, { "id": "context-rot", "kind": "tip", "kindLabel": "Tip", "title": "Mind the middle of long contexts", "body": "Models attend strongly to the beginning and end of a long prompt and get foggy in the middle. Put the instructions and the most important facts at the edges. If something keeps getting ignored, it's probably buried in the soft center.", "tag": "prompting", "date": "2025 · 08 · 12", "publishedAt": "2025-08-12", "note": "Sometimes called 'lost in the middle'.", "extended": [ "Retrieval accuracy as a function of position tends to be U-shaped: strong at the start, strong at the end, sagging in the middle. A fact you need can be technically present and still effectively invisible.", "Design around it. Lead with the instruction, restate the critical constraint at the very end, and keep the soft middle for material you can afford to lose." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/context-rot" }, { "id": "activation-patching", "kind": "method", "kindLabel": "Method", "title": "Activation patching to find the cause", "body": "Run the model twice, once on a clean prompt and once on a corrupted one, then copy a single activation from one run into the other. If the output flips, you just found a component that carries the behavior. It's causal, not just correlational.", "tag": "interpretability", "date": "2025 · 07 · 30", "publishedAt": "2025-07-30", "note": "The closest thing in this field to a controlled experiment.", "extended": [ "Probes tell you what is represented. Patching tells you what matters. By swapping one activation and watching the effect, you move from 'this information is present' to 'this component causes the behavior'.", "Sweep the patch across layers and positions to localize the circuit. The spots where patching flips the answer are the load-bearing ones; everything else is along for the ride." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/activation-patching" }, { "id": "few-shot-format", "kind": "tip", "kindLabel": "Tip", "title": "Your examples teach format louder than content", "body": "In few-shot prompts the model copies the shape of your examples first: the punctuation, the casing, the length, the label words. Get the format dead consistent and the task often solves itself. Sloppy examples teach sloppiness.", "tag": "prompting", "date": "2025 · 07 · 18", "publishedAt": "2025-07-18", "note": null, "extended": [ "Few-shot learning is pattern continuation. The model is extremely sensitive to surface form, sometimes more than to the actual reasoning you intended to demonstrate.", "Make every example identical in structure, pick label words that are single clean tokens, and keep lengths similar. The more rigid the template, the more reliably the model fills in the next blank." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/few-shot-format" }, { "id": "logit-lens", "kind": "method", "kindLabel": "Method", "title": "The logit lens: decode the middle layers", "body": "Take the hidden state from an intermediate layer and run it through the model's output head early. You get the model's 'best guess so far' at that depth. Watching the prediction sharpen layer by layer is like a time-lapse of a thought forming.", "tag": "interpretability", "date": "2025 · 07 · 05", "publishedAt": "2025-07-05", "note": null, "extended": [ "The output projection is just a linear map, so nothing stops you from applying it to an earlier layer. The result is a rough readout of what the model would say if it had to stop thinking right now.", "Often the correct answer appears in the middle layers and the later layers spend their budget refining phrasing and suppressing alternatives. Seeing that handoff demystifies a lot of model behavior." ], "url": "https://ai-insights-for-humans.platphormnews.com/entries/logit-lens" } ] }