diff --git a/extensions/json/server/src/jsonServerMain.ts b/extensions/json/server/src/jsonServerMain.ts
index 52b1468f6f6..f064abed950 100644
--- a/extensions/json/server/src/jsonServerMain.ts
+++ b/extensions/json/server/src/jsonServerMain.ts
@@ -276,6 +276,12 @@ connection.onDidChangeWatchedFiles((change) => {
 });
 
 let jsonDocuments = getLanguageModelCache<JSONDocument>(10, 60, document => languageService.parseJSONDocument(document));
+documents.onDidClose(e => {
+	jsonDocuments.onDocumentRemoved(e.document);
+});
+connection.onShutdown(() => {
+	jsonDocuments.dispose();
+});
 
 function getJSONDocument(document: TextDocument): JSONDocument {
 	return jsonDocuments.get(document);