# On Google Colab, install the scikit-learn version the course was built with.
# On your own computer, install requirements.txt instead; this cell then does nothing.
import subprocess, sys
if "google.colab" in sys.modules:
    subprocess.run([sys.executable, "-m", "pip", "install", "-q", "scikit-learn==1.9.0"], check=True)

Lab 3b. Detecting phishing URLs from their text

This lab classifies URLs as phishing or legitimate using only the characters of the URL. It compares logistic regression and multinomial naive Bayes from Unit 3, each on word tokens and on character n-grams (TF-IDF weights for logistic regression, counts for naive Bayes), shows which n-grams carry the most weight in the character n-gram logistic regression, and examines the URLs that this model gets wrong or nearly wrong. It runs in under a minute on a laptop CPU.

You will:

1. load the URLs and remove exact duplicates;
2. build word-token and character n-gram features;
3. train and compare the models over five random splits;
4. list the n-grams with the largest weights;
5. inspect the legitimate URLs with the highest scores and the phishing URLs with the lowest;
6. test on a brand whose name never appears in the training phishing URLs.

The data is synthetic. data/make_dataset.py builds every URL from templates. Legitimate URLs combine ordinary words and top-level domains, and phishing URLs imitate ten brands with lookalike hosts, hyphenated names, raw IP addresses and login paths. Some legitimate URLs also have login or account pages. Real phishing URLs are far more varied, so the scores here are much higher than you should expect on live data; the point of the lab is the method and the error analysis.

import os
import warnings

import numpy as np
import pandas as pd

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import average_precision_score, confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.pipeline import make_pipeline

warnings.filterwarnings("ignore")
pd.set_option("display.max_colwidth", 80)

LOCAL = "../../data/urls.csv"
URL = "https://raw.githubusercontent.com/aalbata/machine-learning-for-cybersecurity/main/data/urls.csv"
urls = pd.read_csv(LOCAL if os.path.exists(LOCAL) else URL)
n_raw = len(urls)
urls = urls.drop_duplicates("url").reset_index(drop=True)
print(f"{n_raw:,} URLs, {n_raw - len(urls)} exact duplicates removed, {len(urls):,} left")
print(f"phishing share: {urls['label'].mean():.2%}")
urls.sample(8, random_state=3)

6,000 URLs, 33 exact duplicates removed, 5,967 left
phishing share: 9.33%

1. Two ways to turn a URL into features

A word tokenizer splits the URL at punctuation. Scikit-learn's default token pattern keeps runs of two or more letters, digits or underscores, so it splits paypal-verify.xyz into paypal, verify and xyz, and it drops single characters. Character n-grams instead slide a window of \(n\) characters along the string, so they also see fragments such as l-ve, .com. or 0.1, and they still work when an attacker replaces a letter with a digit.

example = ["http://paypa1-verify.xyz/secure/signin?id=482913"]
words = CountVectorizer().fit(example)
chars = CountVectorizer(analyzer="char", ngram_range=(3, 3)).fit(example)
print("word tokens:", list(words.get_feature_names_out()))
print("first character 3-grams:", list(chars.get_feature_names_out())[:15])
print("number of distinct character 3-grams:", len(chars.get_feature_names_out()))

word tokens: ['482913', 'http', 'id', 'paypa1', 'secure', 'signin', 'verify', 'xyz']
first character 3-grams: ['-ve', '.xy', '//p', '/pa', '/se', '/si', '1-v', '291', '482', '829', '913', '://', '=48', '?id', 'a1-']
number of distinct character 3-grams: 46

2. Comparing the models

Four pipelines, each scored over five stratified 70/30 splits:

• logistic regression on TF-IDF weights of character 3- to 5-grams;
• logistic regression on TF-IDF weights of word tokens;
• multinomial naive Bayes on counts of word tokens;
• multinomial naive Bayes on counts of character 3- to 5-grams.

The vectorizers are inside the pipelines, so the vocabulary and the IDF weights are learned from the training URLs only.

def char_tfidf_lr():
    return make_pipeline(TfidfVectorizer(analyzer="char", ngram_range=(3, 5), sublinear_tf=True, min_df=2),
                         LogisticRegression(C=10, max_iter=3000))


PIPES = {
    "char TF-IDF + logistic regression": char_tfidf_lr,
    "word TF-IDF + logistic regression": lambda: make_pipeline(TfidfVectorizer(), LogisticRegression(C=10, max_iter=3000)),
    "word counts + multinomial NB": lambda: make_pipeline(CountVectorizer(), MultinomialNB(alpha=1.0)),
    "char counts + multinomial NB": lambda: make_pipeline(CountVectorizer(analyzer="char", ngram_range=(3, 5), min_df=2),
                                                         MultinomialNB(alpha=1.0)),
}
rows = []
for s in range(5):
    tr, te = train_test_split(urls, test_size=0.3, stratify=urls["label"], random_state=s)
    for name, make in PIPES.items():
        p = make().fit(tr["url"], tr["label"]).predict_proba(te["url"])[:, 1]
        tn, fp, fn, tp = confusion_matrix(te["label"], p >= 0.5).ravel()
        rows.append(dict(model=name, seed=s, ap=average_precision_score(te["label"], p),
                         recall=tp / (tp + fn), precision=tp / max(tp + fp, 1), fp=fp, fn=fn))
runs = pd.DataFrame(rows)
table = runs.groupby("model", sort=False).agg(["mean", "std"]).drop(columns="seed")
print(table.round(4).to_string())

                                       ap          recall         precision        fp        fn        
                                     mean     std    mean     std      mean  std mean  std mean     std
model                                                                                                  
char TF-IDF + logistic regression  0.9977  0.0014  0.9641  0.0060       1.0  0.0  0.0  0.0  6.0  1.0000
word TF-IDF + logistic regression  0.9981  0.0008  0.9641  0.0164       1.0  0.0  0.0  0.0  6.0  2.7386
word counts + multinomial NB       0.9962  0.0017  0.9593  0.0107       1.0  0.0  0.0  0.0  6.8  1.7889
char counts + multinomial NB       0.9972  0.0017  0.9808  0.0066       1.0  0.0  0.0  0.0  3.2  1.0954

All four models separate the classes almost perfectly on this synthetic data, and at the default threshold of 0.5 none of them raises a false alert. Their differences are in recall, and the cell above shows how many phishing URLs each misses.

3. Which n-grams carry the weight?

For the character model trained on the first split, the largest positive coefficients point toward phishing and the largest negative ones toward legitimate URLs.

tr, te = train_test_split(urls, test_size=0.3, stratify=urls["label"], random_state=0)
model = char_tfidf_lr().fit(tr["url"], tr["label"])
vec, clf = model[0], model[-1]
names = vec.get_feature_names_out()
w = clf.coef_[0]
order = np.argsort(w)
top_phish = [(names[i], float(w[i])) for i in order[::-1][:15]]
top_legit = [(names[i], float(w[i])) for i in order[:15]]
print("toward phishing:  ", ", ".join(f"'{n}' {v:.2f}" for n, v in top_phish))
print("toward legitimate:", ", ".join(f"'{n}' {v:.2f}" for n, v in top_legit))
print("vocabulary size:", len(names))

toward phishing:   '?id=' 2.81, '?id' 2.81, 'id=' 2.81, 'eri' 2.75, 'ify' 2.75, 'veri' 2.75, 'erify' 2.75, 'erif' 2.75, 'rif' 2.75, 'ver' 2.75, 'verif' 2.75, 'rify' 2.75, 'ex.ph' 2.62, 'ex.p' 2.62, 'x.ph' 2.62
toward legitimate: 'www.' -2.84, 'www' -2.84, '/ww' -2.84, 'ww.' -2.84, '/www.' -2.84, '/www' -2.84, '://ww' -2.84, '//ww' -2.84, '//www' -2.84, '//w' -2.72, '://w' -2.72, 's://w' -2.53, 'ear' -1.58, '.ae/' -1.57, '.ae' -1.57
vocabulary size: 10572

N-grams that always occur together, such as the overlapping pieces of verify, receive identical weights, so read the list as a few distinct patterns rather than fifteen. Several of the strongest patterns are artifacts of how the templates were written, for example the ?id= query string that only phishing URLs carry and the www. prefix that only legitimate URLs carry. A model trained on real data can learn the same kind of shortcut from the way the data was collected (Arp et al., 2022, call this a spurious correlation), and an attacker who drops the query string or adds www. evades it at no cost. That is why the error analysis below matters.

4. Error analysis

Instead of looking only at errors at one threshold, we look at the legitimate URLs with the highest phishing scores (the false alerts you would get first as the threshold is lowered) and the phishing URLs with the lowest scores (the first to be missed as it is raised).

te = te.assign(score=model.predict_proba(te["url"])[:, 1])
legit_top = te[te["label"] == 0].sort_values("score", ascending=False).head(10)
phish_low = te[te["label"] == 1].sort_values("score").head(10)
print("legitimate URLs with the highest scores")
print(legit_top[["url", "score"]].to_string(index=False))
print()
print("phishing URLs with the lowest scores")
print(phish_low[["url", "score"]].to_string(index=False))
login_words = r"/(login|account|signin)"
share_login_top = float(legit_top["url"].str.contains(login_words).mean())
share_login_all = float(te.loc[te["label"] == 0, "url"].str.contains(login_words).mean())
print(f"\nshare with a login, account or signin path: {share_login_top:.0%} of these ten, {share_login_all:.1%} of all legitimate test URLs")
tn, fp, fn, tp = confusion_matrix(te["label"], te["score"] >= 0.5).ravel()
print(f"at threshold 0.5 on this split: {fp} false positives, {fn} false negatives out of {tp + fn} phishing URLs")

legitimate URLs with the highest scores
                              url    score
       http://booksshop.net/login 0.164278
 https://recipesweather.edu/login 0.125246
    http://schoolhealth.com/login 0.100814
        https://jobsshop.io/login 0.098499
  https://photo-recipes.edu/login 0.084556
 http://gardennews.com/login/5771 0.083994
    https://recipesmaps.edu/login 0.080613
  https://recipestravel.edu/login 0.079348
http://sports-maps.com/login/4515 0.077251
  https://photomaps.net/login/603 0.075883

phishing URLs with the lowest scores
                                      url    score
            https://sportsschool.ru/login 0.162952
https://markethealth.edu/chase/login.html 0.301911
              https://bloglearn.ae/verify 0.350916
              https://76.87.173.209/login 0.370207
 https://healthsports.net/login?id=116371 0.401369
          https://travelcity.online/login 0.483407
           http://booksbooks.io/index.php 0.506992
           http://booksblog.edu/index.php 0.537911
                 https://cut33.com/@login 0.576374
               http://blognews.live/login 0.615209

share with a login, account or signin path: 100% of these ten, 28.8% of all legitimate test URLs
at threshold 0.5 on this split: 0 false positives, 6 false negatives out of 167 phishing URLs

The legitimate URLs that come closest to an alert are pages with login, account or signin paths (the cell prints their share among these ten and among all legitimate test URLs), which the phishing templates also use. The phishing URLs that score lowest are those whose host looks ordinary, such as a host made of two common words, usually with no brand name anywhere in the URL, or a raw IP address. On this split, five of the six phishing URLs missed at 0.5 still score above every legitimate test URL, so a somewhat lower threshold would catch them without a false alert. Only the lowest-scoring one falls below a legitimate login page, and catching it would also produce a false alert on that page.

5. A brand the model has not seen

A detector meets new targets all the time. Here we drop from the training split every phishing URL that contains one brand name and test on the phishing URLs of the test split that contain it, together with all legitimate test URLs. Lookalike spellings such as payp4l do not contain the name and stay in the training data.

brand = "paypal"
mentions = urls["url"].str.contains(brand)
tr, te = train_test_split(urls, test_size=0.3, stratify=urls["label"], random_state=0)
tr_nb = tr[~(mentions.loc[tr.index] & (tr["label"] == 1))]
te_brand = te[(te["label"] == 0) | mentions.loc[te.index]]
unseen = {}
for name in ["char TF-IDF + logistic regression", "word counts + multinomial NB"]:
    p = PIPES[name]().fit(tr_nb["url"], tr_nb["label"]).predict_proba(te_brand["url"])[:, 1]
    y = te_brand["label"].to_numpy()
    unseen[name] = dict(ap=average_precision_score(y, p), recall=float((p[y == 1] >= 0.5).mean()), n=int(y.sum()))
    print(f"{name:<36} AP {unseen[name]['ap']:.3f}, recall at 0.5 {unseen[name]['recall']:.3f} on {unseen[name]['n']} unseen-brand phishing URLs")

char TF-IDF + logistic regression    AP 1.000, recall at 0.5 1.000 on 8 unseen-brand phishing URLs
word counts + multinomial NB         AP 1.000, recall at 0.5 1.000 on 8 unseen-brand phishing URLs

The test set contains only a few phishing URLs for the held-out brand (the cell prints how many), so this is a small test. The models score them well because the templates reuse the same hosting patterns and paths for every brand. Real campaigns change those patterns too, so this is an optimistic test.

Try this

• Change ngram_range to (1, 2) for the character model. Which kinds of URL does it start to miss?
• Set alpha=0.01 in the naive Bayes models. What happens to the scores of URLs that contain a token or n-gram seen in the training URLs of only one class? (Tokens and n-grams never seen in training are not in the vocabulary, so the vectorizer drops them and alpha does not affect them.)
• Hold out a different brand, for example "dhl", and compare.

Results summary

These are the numbers quoted in the lesson text and the data used for the figures.

short = {"char TF-IDF + logistic regression": "char_lr", "word TF-IDF + logistic regression": "word_lr",
         "word counts + multinomial NB": "word_nb", "char counts + multinomial NB": "char_nb"}
results = {"n_urls_raw": int(n_raw), "n_urls": int(len(urls)), "n_url_duplicates": int(n_raw - len(urls)),
           "phish_share": float(urls["label"].mean()), "vocab_char": int(len(names)),
           "share_login_top10": share_login_top, "share_login_all": share_login_all,
           "fp_split0": int(fp), "fn_split0": int(fn), "n_phish_test0": int(tp + fn)}
for name, s in short.items():
    for col in ["ap", "recall", "precision", "fp", "fn"]:
        results[f"{col}_{s}"] = float(table.loc[name, (col, "mean")])
        results[f"{col}_{s}_sd"] = float(table.loc[name, (col, "std")])
for name, s in [("char TF-IDF + logistic regression", "char_lr"), ("word counts + multinomial NB", "word_nb")]:
    results[f"unseen_ap_{s}"] = float(unseen[name]["ap"])
    results[f"unseen_recall_{s}"] = float(unseen[name]["recall"])
results["unseen_n"] = int(unseen["word counts + multinomial NB"]["n"])
for k_, v in results.items():
    print(f"{k_:<24} {v}")

export = {"top_phish": top_phish, "top_legit": top_legit,
          "legit_top": [[u, float(s)] for u, s in zip(legit_top["url"], legit_top["score"])],
          "phish_low": [[u, float(s)] for u, s in zip(phish_low["url"], phish_low["score"])]}

# When the course build runs this notebook, it saves the numbers for the lesson pages.
if os.environ.get("COURSE_RESULTS"):
    import json
    with open(os.environ["COURSE_RESULTS"], "w") as f:
        json.dump({"results": results, "data": export}, f)

n_urls_raw               6000
n_urls                   5967
n_url_duplicates         33
phish_share              0.09334674040556394
vocab_char               10572
share_login_top10        1.0
share_login_all          0.2875615763546798
fp_split0                0
fn_split0                6
n_phish_test0            167
ap_char_lr               0.9977220550067234
ap_char_lr_sd            0.0014478973079731067
recall_char_lr           0.9640718562874252
recall_char_lr_sd        0.0059880239520958105
precision_char_lr        1.0
precision_char_lr_sd     0.0
fp_char_lr               0.0
fp_char_lr_sd            0.0
fn_char_lr               6.0
fn_char_lr_sd            1.0
ap_word_lr               0.9981194999660259
ap_word_lr_sd            0.0008065485150803181
recall_word_lr           0.9640718562874252
recall_word_lr_sd        0.016398878967220566
precision_word_lr        1.0
precision_word_lr_sd     0.0
fp_word_lr               0.0
fp_word_lr_sd            0.0
fn_word_lr               6.0
fn_word_lr_sd            2.7386127875258306
ap_word_nb               0.9961883569425798
ap_word_nb_sd            0.0017491109573900742
recall_word_nb           0.9592814371257485
recall_word_nb_sd        0.01071170288622651
precision_word_nb        1.0
precision_word_nb_sd     0.0
fp_word_nb               0.0
fp_word_nb_sd            0.0
fn_word_nb               6.8
fn_word_nb_sd            1.7888543819998317
ap_char_nb               0.9972218998552753
ap_char_nb_sd            0.0016739887475073884
recall_char_nb           0.9808383233532935
recall_char_nb_sd        0.0065595515868881885
precision_char_nb        1.0
precision_char_nb_sd     0.0
fp_char_nb               0.0
fp_char_nb_sd            0.0
fn_char_nb               3.2
fn_char_nb_sd            1.0954451150103321
unseen_ap_char_lr        1.0
unseen_recall_char_lr    1.0
unseen_ap_word_nb        1.0
unseen_recall_word_nb    1.0
unseen_n                 8