Moves tokenization logic from text model to its own text model part.

This commit is contained in:
Henning Dieterichs
2022-04-01 17:35:06 +02:00
parent e8f26b998e
commit 6e1e0b1d89
78 changed files with 1006 additions and 743 deletions

View File

@@ -120,7 +120,7 @@ class ModelWorkerTextMateTokenizer extends Disposable {
}
}
this._model.setTokens(tokens);
this._model.tokenization.setTokens(tokens);
}
}