# On Google Colab, install the scikit-learn version the course was built with.
# On your own computer, install requirements.txt instead; this cell then does nothing.
import subprocess, sys
if "google.colab" in sys.modules:
    subprocess.run([sys.executable, "-m", "pip", "install", "-q", "scikit-learn==1.9.0"], check=True)

Lab 6b. Detecting anomalous log sequences with an LSTM

This lab builds a next-event model of system logs and uses it to flag anomalous sequences. It follows the idea behind DeepLog (Lesson 6.2), which learns what event usually follows a short history of events and flags any event that falls outside the model's top predictions. We compare a count-based n-gram baseline with a small LSTM, on synthetic logs with several kinds of injected anomalies, and report precision and recall.

You will:

1. generate normal log-event sequences from a stochastic workflow;
2. inject three kinds of anomalous sequences;
3. train an n-gram next-event model and an LSTM next-event model on normal data only;
4. flag a sequence if any event falls outside the model's top-g predictions;
5. report precision and recall for several g, and which anomaly kinds each method catches.

The logs are synthetic. Public labeled security-log datasets are scarce, their templates are system specific, and a generator lets us control exactly which anomalies are present so that precision and recall are meaningful. The workflow below is a deliberately simple model of the activity of one user between login and logout and is not drawn from any real system. The lab runs in about a minute on a CPU.

import os
import warnings

import numpy as np
import torch
import torch.nn as nn

warnings.filterwarnings("ignore", category=UserWarning)
torch.set_num_threads(4)
SEED = 20251015
torch.manual_seed(SEED)
rng = np.random.default_rng(SEED)

1. A workflow that emits log events

Each sequence is a sequence of event templates drawn from a small vocabulary. A user logs in, then performs a mix of file access, directory listing and, for some sequences, an authorized privileged action, then logs out. The next event depends on the current one, so the sequence has structure that a model can learn. We draw sequences from a first-order Markov chain over the events, with a dedicated start and end.

EVENTS = ["LOGIN", "AUTH_OK", "LIST_DIR", "READ_FILE", "WRITE_FILE",
          "REQUEST_PRIV", "GRANT_PRIV", "RUN_PRIV", "LOGOUT"]
E = {name: i for i, name in enumerate(EVENTS)}
V = len(EVENTS)
BOS, EOS = V, V + 1  # begin and end markers
VOCAB = V + 2

# Transition probabilities for the normal workflow. Rows are the current event, columns the next.
P = {
    "BOS": {"LOGIN": 1.0},
    "LOGIN": {"AUTH_OK": 1.0},
    "AUTH_OK": {"LIST_DIR": 0.5, "READ_FILE": 0.3, "REQUEST_PRIV": 0.2},
    "LIST_DIR": {"READ_FILE": 0.5, "LIST_DIR": 0.2, "WRITE_FILE": 0.1, "LOGOUT": 0.2},
    "READ_FILE": {"READ_FILE": 0.4, "LIST_DIR": 0.2, "WRITE_FILE": 0.2, "LOGOUT": 0.2},
    "WRITE_FILE": {"READ_FILE": 0.3, "WRITE_FILE": 0.2, "LOGOUT": 0.5},
    "REQUEST_PRIV": {"GRANT_PRIV": 1.0},
    "GRANT_PRIV": {"RUN_PRIV": 1.0},
    "RUN_PRIV": {"READ_FILE": 0.3, "WRITE_FILE": 0.3, "LOGOUT": 0.4},
    "LOGOUT": {"EOS": 1.0},
}


def sample_normal():
    seq = [BOS]
    cur = "LOGIN"
    seq.append(E[cur])
    steps = 0
    while cur != "LOGOUT" and steps < 30:
        nxt = P[cur]
        names = list(nxt)
        probs = np.array([nxt[n] for n in names])
        cur = names[rng.choice(len(names), p=probs)]
        if cur == "EOS":
            break
        seq.append(E[cur])
        steps += 1
    if seq[-1] != E["LOGOUT"]:
        seq.append(E["LOGOUT"])
    seq.append(EOS)
    return seq


normal_seqs = [sample_normal() for _ in range(6000)]
print(f"generated {len(normal_seqs)} normal sequences")
print("example:", " ".join(EVENTS[i] if i < V else ("BOS" if i == BOS else "EOS") for i in normal_seqs[0]))

generated 6000 normal sequences
example: BOS LOGIN AUTH_OK LIST_DIR LOGOUT EOS

2. Injecting anomalies

We build anomalous sequences of three kinds, each a plausible deviation an intruder might cause.

• In out_of_order, two adjacent, different events of a normal sequence are swapped, for example a file written before the directory listing that should precede it. A swap is kept only if the resulting sequence contains at least one transition that the workflow cannot produce; otherwise the swapped sequence would be an ordinary normal sequence carrying an anomaly label.
• In rare_transition, the privileged step RUN_PRIV is repeated three to five times directly after authentication, a transition the normal workflow never produces.
• In priv_without_auth, a privileged action (RUN_PRIV) is reached without the preceding REQUEST_PRIV and GRANT_PRIV, modeling a privilege escalation that skips authorization.

def event_name(i):
    return "BOS" if i == BOS else ("EOS" if i == EOS else EVENTS[i])


def workflow_can_produce(seq):
    """True if every transition in seq has positive probability in the normal workflow."""
    return all(P[event_name(a)].get(event_name(b), 0.0) > 0 for a, b in zip(seq[:-1], seq[1:]))


def make_out_of_order():
    while True:
        s = sample_normal()
        positions = [i for i in range(2, len(s) - 3) if s[i] != s[i + 1]]
        rng.shuffle(positions)
        for i in positions:
            t = list(s)
            t[i], t[i + 1] = t[i + 1], t[i]
            if not workflow_can_produce(t):
                return t


def make_rare_transition():
    s = [BOS, E["LOGIN"], E["AUTH_OK"]]
    for _ in range(rng.integers(3, 6)):
        s.append(E["RUN_PRIV"])  # running a privileged step repeatedly without ever requesting it
    s += [E["LOGOUT"], EOS]
    return s


def make_priv_without_auth():
    s = [BOS, E["LOGIN"], E["AUTH_OK"], E["LIST_DIR"]]
    s += [E["RUN_PRIV"], E["WRITE_FILE"], E["LOGOUT"], EOS]
    return s


KINDS = ["out_of_order", "rare_transition", "priv_without_auth"]
MAKERS = {"out_of_order": make_out_of_order,
          "rare_transition": make_rare_transition,
          "priv_without_auth": make_priv_without_auth}

n_anom_each = 200
anom_seqs, anom_kinds = [], []
for k in KINDS:
    for _ in range(n_anom_each):
        anom_seqs.append(MAKERS[k]())
        anom_kinds.append(k)
print(f"generated {len(anom_seqs)} anomalous sequences, {n_anom_each} of each kind")

generated 600 anomalous sequences, 200 of each kind

3. Train and test sets

The models see only normal sequences during training, as a detector would when normal behavior is known but attacks are not. The test set mixes held-out normal sequences with the anomalies.

split = 4000
train_seqs = normal_seqs[:split]
test_normal = normal_seqs[split:]
test_seqs = test_normal + anom_seqs
test_labels = np.array([0] * len(test_normal) + [1] * len(anom_seqs))
test_kinds = ["normal"] * len(test_normal) + anom_kinds
print(f"train: {len(train_seqs)} normal; test: {len(test_normal)} normal + {len(anom_seqs)} anomalous")

train: 4000 normal; test: 2000 normal + 600 anomalous

4. An n-gram next-event model

The baseline estimates the probability of the next event from the preceding \(w\) events (W in the code) by counting how often each continuation appears in the training data. A sequence is flagged if, at any position, the true next event is not among the g most frequent continuations of its history. Unseen histories are treated as predicting nothing, so their next event always falls outside the top g. Only continuations observed after a history can be among its top g, so a history with fewer than g observed continuations has fewer than g candidates.

W = 3  # history length w


def ngram_counts(seqs, w):
    table = {}
    for s in seqs:
        for t in range(len(s) - 1):
            ctx = tuple(s[max(0, t - w + 1):t + 1])
            nxt = s[t + 1]
            table.setdefault(ctx, np.zeros(VOCAB))
            table[ctx][nxt] += 1
    return table


ngram = ngram_counts(train_seqs, W)


def ngram_topg(ctx, g):
    counts = ngram.get(ctx)
    if counts is None or counts.sum() == 0:
        return set()
    seen = np.flatnonzero(counts)  # continuations observed after this history
    ranked = seen[np.argsort(-counts[seen], kind="stable")]
    return set(ranked[:g].tolist())


def ngram_flags(seq, g):
    for t in range(len(seq) - 1):
        ctx = tuple(seq[max(0, t - W + 1):t + 1])
        if seq[t + 1] not in ngram_topg(ctx, g):
            return True
    return False

5. An LSTM next-event model

The LSTM reads the history of events and outputs a probability distribution over the next event. Unlike the n-gram model, it shares information across histories through its hidden state, so a history it never saw exactly can still receive a sensible prediction. The n-gram model looks only at the last \(w\) events (W in the code), whereas the LSTM reads the whole prefix of the sequence. We train it to predict each event from those before it, on normal sequences only.

class NextEvent(nn.Module):
    def __init__(self, vocab=VOCAB, emb=24, hidden=48):
        super().__init__()
        self.emb = nn.Embedding(vocab, emb)
        self.lstm = nn.LSTM(emb, hidden, batch_first=True)
        self.out = nn.Linear(hidden, vocab)

    def forward(self, x):
        e = self.emb(x)
        h, _ = self.lstm(e)
        return self.out(h)


def make_batches(seqs, batch=128):
    order = rng.permutation(len(seqs))
    for i in range(0, len(seqs), batch):
        chunk = [seqs[j] for j in order[i:i + batch]]
        maxlen = max(len(s) for s in chunk)
        X = np.full((len(chunk), maxlen - 1), EOS, dtype=np.int64)
        Y = np.full((len(chunk), maxlen - 1), -100, dtype=np.int64)
        for r, s in enumerate(chunk):
            X[r, :len(s) - 1] = s[:-1]
            Y[r, :len(s) - 1] = s[1:]
        yield torch.from_numpy(X), torch.from_numpy(Y)


model = NextEvent()
opt = torch.optim.Adam(model.parameters(), lr=5e-3)
loss_fn = nn.CrossEntropyLoss(ignore_index=-100)
EPOCHS = 6
for ep in range(EPOCHS):
    model.train()
    total = 0.0
    for X, Y in make_batches(train_seqs):
        opt.zero_grad()
        logits = model(X)
        loss = loss_fn(logits.reshape(-1, VOCAB), Y.reshape(-1))
        loss.backward()
        opt.step()
        total += loss.item()
    print(f"epoch {ep + 1}: mean batch loss {total / (len(train_seqs) // 128 + 1):.4f}")


@torch.no_grad()
def lstm_flags(seq, g):
    model.eval()
    x = torch.tensor(seq[:-1], dtype=torch.long).unsqueeze(0)
    logits = model(x)[0]
    topg = logits.topk(g, dim=-1).indices.numpy()
    targets = np.array(seq[1:])
    for t in range(len(targets)):
        if targets[t] not in topg[t]:
            return True
    return False

epoch 1: mean batch loss 1.4204

epoch 2: mean batch loss 0.7393

epoch 3: mean batch loss 0.7078

epoch 4: mean batch loss 0.7037

epoch 5: mean batch loss 0.7042

epoch 6: mean batch loss 0.7011

6. Precision and recall for several g

A larger g makes each model more permissive, because more continuations count as normal and fewer sequences are flagged. We sweep g and report precision and recall for both models, treating a flagged sequence as a positive prediction.

def evaluate(flag_fn, g):
    preds = np.array([1 if flag_fn(s, g) else 0 for s in test_seqs])
    tp = int(((preds == 1) & (test_labels == 1)).sum())
    fp = int(((preds == 1) & (test_labels == 0)).sum())
    fn = int(((preds == 0) & (test_labels == 1)).sum())
    precision = tp / (tp + fp) if tp + fp else 0.0
    recall = tp / (tp + fn) if tp + fn else 0.0
    by_kind = {}
    for k in KINDS:
        mask = np.array([kk == k for kk in test_kinds])
        by_kind[k] = float(preds[mask].mean())
    return precision, recall, by_kind, preds


G_VALUES = [1, 2, 3, 4]
ngram_rows, lstm_rows = {}, {}
for g in G_VALUES:
    pn, rn, kn, _ = evaluate(ngram_flags, g)
    pl, rl, kl, _ = evaluate(lstm_flags, g)
    ngram_rows[g] = (pn, rn, kn)
    lstm_rows[g] = (pl, rl, kl)
    print(f"g={g}  n-gram P={pn:.3f} R={rn:.3f}   LSTM P={pl:.3f} R={rl:.3f}")

g=1  n-gram P=0.231 R=1.000   LSTM P=0.231 R=1.000

g=2  n-gram P=0.253 R=1.000   LSTM P=0.285 R=1.000

g=3  n-gram P=0.431 R=1.000   LSTM P=0.462 R=1.000

g=4  n-gram P=1.000 R=1.000   LSTM P=1.000 R=0.953

Recall per anomaly kind shows which deviations each model catches. The cell below prints it at g = 2 and at g = 4, the largest number of successors any event has in the workflow.

_, _, kn2, _ = evaluate(ngram_flags, 2)
_, _, kl2, _ = evaluate(lstm_flags, 2)
_, _, kn4, _ = evaluate(ngram_flags, 4)
_, _, kl4, _ = evaluate(lstm_flags, 4)
for k in KINDS:
    print(f"{k:<18} g=2: n-gram {kn2[k]:.2f}  LSTM {kl2[k]:.2f}   g=4: n-gram {kn4[k]:.2f}  LSTM {kl4[k]:.2f}")

out_of_order       g=2: n-gram 1.00  LSTM 1.00   g=4: n-gram 1.00  LSTM 0.86
rare_transition    g=2: n-gram 1.00  LSTM 1.00   g=4: n-gram 1.00  LSTM 1.00
priv_without_auth  g=2: n-gram 1.00  LSTM 1.00   g=4: n-gram 1.00  LSTM 1.00

Try this

• Change the history length W of the n-gram model to 1 and predict, before running it, whether its recall on out-of-order anomalies can change (every kept swap contains a transition that never occurs in training); then compare its precision at g = 2 and g = 3 with the values above.
• Add a fourth anomaly kind that repeats a normal event many more times than usual and check which model notices.
• Reduce the training set to 500 sequences and compare how the two models degrade.

Results summary

These are the numbers quoted in the lesson text.

def row(d, g):
    p, r, _ = d[g]
    return float(p), float(r)


results = {
    "n_train": len(train_seqs),
    "n_test_normal": len(test_normal),
    "n_test_anom": len(anom_seqs),
    "n_anom_each": n_anom_each,
    "history": W,
    "vocab": V,
    "ngram_p_g1": row(ngram_rows, 1)[0], "ngram_r_g1": row(ngram_rows, 1)[1],
    "ngram_p_g2": row(ngram_rows, 2)[0], "ngram_r_g2": row(ngram_rows, 2)[1],
    "lstm_p_g1": row(lstm_rows, 1)[0], "lstm_r_g1": row(lstm_rows, 1)[1],
    "lstm_p_g2": row(lstm_rows, 2)[0], "lstm_r_g2": row(lstm_rows, 2)[1],
    "lstm_p_g3": row(lstm_rows, 3)[0], "lstm_r_g3": row(lstm_rows, 3)[1],
    "lstm_p_g4": row(lstm_rows, 4)[0], "lstm_r_g4": row(lstm_rows, 4)[1],
    "ngram_p_g4": row(ngram_rows, 4)[0], "ngram_r_g4": row(ngram_rows, 4)[1],
    "ngram_p_g3": row(ngram_rows, 3)[0], "ngram_r_g3": row(ngram_rows, 3)[1],
    "ngram_recall_ooo_g2": float(kn2["out_of_order"]),
    "ngram_recall_rare_g2": float(kn2["rare_transition"]),
    "ngram_recall_priv_g2": float(kn2["priv_without_auth"]),
    "lstm_recall_ooo_g2": float(kl2["out_of_order"]),
    "lstm_recall_rare_g2": float(kl2["rare_transition"]),
    "lstm_recall_priv_g2": float(kl2["priv_without_auth"]),
    "ngram_recall_ooo_g4": float(kn4["out_of_order"]),
    "ngram_recall_rare_g4": float(kn4["rare_transition"]),
    "ngram_recall_priv_g4": float(kn4["priv_without_auth"]),
    "lstm_recall_ooo_g4": float(kl4["out_of_order"]),
    "lstm_recall_rare_g4": float(kl4["rare_transition"]),
    "lstm_recall_priv_g4": float(kl4["priv_without_auth"]),
}
for k, v in results.items():
    print(f"{k:<22} {v}")

export = {
    "g_values": G_VALUES,
    "ngram_precision": [row(ngram_rows, g)[0] for g in G_VALUES],
    "ngram_recall": [row(ngram_rows, g)[1] for g in G_VALUES],
    "lstm_precision": [row(lstm_rows, g)[0] for g in G_VALUES],
    "lstm_recall": [row(lstm_rows, g)[1] for g in G_VALUES],
    "kinds": KINDS,
    "ngram_by_kind_g2": [float(kn2[k]) for k in KINDS],
    "lstm_by_kind_g2": [float(kl2[k]) for k in KINDS],
    "ngram_by_kind_g4": [float(kn4[k]) for k in KINDS],
    "lstm_by_kind_g4": [float(kl4[k]) for k in KINDS],
    "events": EVENTS,
}

if os.environ.get("COURSE_RESULTS"):
    import json
    with open(os.environ["COURSE_RESULTS"], "w") as f:
        json.dump({"results": results, "data": export}, f)

n_train                4000
n_test_normal          2000
n_test_anom            600
n_anom_each            200
history                3
vocab                  9
ngram_p_g1             0.23076923076923078
ngram_r_g1             1.0
ngram_p_g2             0.2528445006321112
ngram_r_g2             1.0
lstm_p_g1              0.23076923076923078
lstm_r_g1              1.0
lstm_p_g2              0.285442435775452
lstm_r_g2              1.0
lstm_p_g3              0.4618937644341801
lstm_r_g3              1.0
lstm_p_g4              1.0
lstm_r_g4              0.9533333333333334
ngram_p_g4             1.0
ngram_r_g4             1.0
ngram_p_g3             0.43103448275862066
ngram_r_g3             1.0
ngram_recall_ooo_g2    1.0
ngram_recall_rare_g2   1.0
ngram_recall_priv_g2   1.0
lstm_recall_ooo_g2     1.0
lstm_recall_rare_g2    1.0
lstm_recall_priv_g2    1.0
ngram_recall_ooo_g4    1.0
ngram_recall_rare_g4   1.0
ngram_recall_priv_g4   1.0
lstm_recall_ooo_g4     0.86
lstm_recall_rare_g4    1.0
lstm_recall_priv_g4    1.0