Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions spacy/tests/vocab_vectors/test_vectors.py
Original file line number Diff line number Diff line change
Expand Up @@ -376,6 +376,48 @@ def test_vocab_prune_vectors():
assert_allclose(float(similarity), cosine, atol=1e-4, rtol=1e-3)


def test_vocab_prune_vectors_duplicate_rows():
# Real vector tables sometimes alias one key onto another key's row
# (e.g. for spelling or casing variants) instead of duplicating the
# vector data. Pruning must not waste a kept slot duplicating such a
# shared row, and the aliased keys must still resolve to their
# partner's vector afterwards, rather than being lost or sent through
# the nearest-neighbour remap on their own.
vocab = Vocab(vectors_name="test_vocab_prune_vectors_duplicate_rows")
_ = vocab["cat"] # noqa: F841
_ = vocab["kitty"] # noqa: F841
_ = vocab["dog"] # noqa: F841
_ = vocab["puppy"] # noqa: F841
_ = vocab["extra"] # noqa: F841
data = OPS.xp.ndarray((3, 3), dtype="f")
data[0] = OPS.asarray([1.0, 1.2, 1.1])
data[1] = OPS.asarray([0.3, 1.3, 1.0])
data[2] = OPS.asarray([0.9, 1.22, 1.05])
vocab.set_vector("cat", data[0])
vocab.set_vector("dog", data[1])
vocab.set_vector("extra", data[2])
vocab.vectors.add(
vocab.strings["kitty"], row=vocab.vectors.key2row[vocab.strings["cat"]]
)
vocab.vectors.add(
vocab.strings["puppy"], row=vocab.vectors.key2row[vocab.strings["dog"]]
)
assert len(set(vocab.vectors.key2row.values())) == 3

remap = vocab.prune_vectors(2, batch_size=2)

# Only "extra" lacked a partner sharing its row, so it's the only one
# without a surviving row of its own once pruned down to 2.
assert vocab.vectors.shape[0] == 2
assert list(remap.keys()) == ["extra"]
neighbour, similarity = remap["extra"]
assert neighbour in ("cat", "kitty"), remap
cosine = get_cosine(data[0], data[2])
assert_allclose(float(similarity), cosine, atol=1e-4, rtol=1e-3)
assert list(vocab["kitty"].vector) == list(vocab["cat"].vector)
assert list(vocab["puppy"].vector) == list(vocab["dog"].vector)


def test_vectors_serialize():
data = OPS.asarray([[4, 2, 2, 2], [4, 2, 2, 2], [1, 1, 1, 1]], dtype="f")
v = Vectors(data=data, keys=["A", "B", "C"])
Expand Down
33 changes: 31 additions & 2 deletions spacy/vocab.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -427,20 +427,49 @@ cdef class Vocab:
if value in self.vectors.key2row:
priority.append((-lex.prob, self.vectors.key2row[value], value))
priority.sort()
indices = xp.asarray([i for (prob, i, key) in priority], dtype="uint64")
keys = xp.asarray([key for (prob, i, key) in priority], dtype="uint64")
# A single row can be shared by more than one key, for example when
# Vectors.add(key, row=...) has aliased one key to another's vector.
# Collapse the priority list down to one entry per row (keeping the
# highest-priority key as that row's representative) before deciding
# what to keep or discard. Without this, a shared row can either end
# up duplicated into `keep`, wasting a slot that could have held a
# distinct vector, or be split across `keep` and `toss`, sending a
# key through the nearest-neighbour search below even though its
# exact vector is still present among the kept rows.
seen_rows = set()
deduped = []
aliases = [] # (row, alias_key) pairs to restore after pruning
for prob, row, key in priority:
if row in seen_rows:
aliases.append((row, key))
continue
seen_rows.add(row)
deduped.append((prob, row, key))
indices = xp.asarray([i for (prob, i, key) in deduped], dtype="uint64")
keys = xp.asarray([key for (prob, i, key) in deduped], dtype="uint64")
keep = xp.ascontiguousarray(self.vectors.data[indices[:nr_row]])
toss = xp.ascontiguousarray(self.vectors.data[indices[nr_row:]])
self.vectors = Vectors(strings=self.strings, data=keep, keys=keys[:nr_row], name=self.vectors.name)
syn_keys, syn_rows, scores = self.vectors.most_similar(toss, batch_size=batch_size)
syn_keys = ops.to_numpy(syn_keys)
remap = {}
tossed_keys = set()
for i, key in enumerate(ops.to_numpy(keys[nr_row:])):
self.vectors.add(key, row=syn_rows[i][0])
word = self.strings[key]
synonym = self.strings[syn_keys[i][0]]
score = scores[i][0]
remap[word] = (synonym, score)
tossed_keys.add(int(key))
# Every alias key shares its row's representative key's fate exactly,
# whether that row was kept in place or remapped to a synonym above,
# so just point it at wherever the representative key ended up.
row_to_key = {row: key for (_prob, row, key) in deduped}
for row, alias_key in aliases:
rep_key = int(row_to_key[row])
self.vectors.add(int(alias_key), row=self.vectors.key2row[rep_key])
if rep_key in tossed_keys:
remap[self.strings[int(alias_key)]] = remap[self.strings[rep_key]]
return remap

def get_vector(self, orth):
Expand Down