feat: switch cache to be memory size based

packages/core/src/util/cache.js

@@ -9,48 +9,68 @@ export const voidCache = () => ({
 });
 
 export function lruCache({
-  max = 1000, // max entries
+  max = 1 * 1024 * 1024, // 1 MB cache size as default
   ttl = 3 * 60 * 60 * 1000 // time-to-live, default 3 hours
 } = {}) {
   let cache = new Map;
+  let currSize = 0;
 
   function evict() {
     const expire = performance.now() - ttl;
-    let lruKey = null;
-    let lruLast = Infinity;
 
-    for (const [key, value] of cache) {
-      const { last } = value;
+    while (currSize > max) {
+      let lruKey = null;
+      let lruLast = Infinity;
+      let lruSize = null;
 
-      // least recently used entry seen so far
-      if (last < lruLast) {
-        lruKey = key;
-        lruLast = last;
+      for (const [key, value] of cache) {
+        const { last, size } = value;
+
+        // least recently used entry seen so far
+        if (last < lruLast) {
+          lruKey = key;
+          lruLast = last;
+          lruSize = size;
+        }
+
+        // remove if time since last access exceeds ttl
+        if (expire > last) {
+          cache.delete(key);
+          currSize -= size;
+        }
       }
-
-      // remove if time since last access exceeds ttl
-      if (expire > last) {
-        cache.delete(key);
+
+      // remove lru entry
+      if (cache.has(lruKey)) {
+        currSize -= lruSize;
+        cache.delete(lruKey);
       }
     }
-
-    // remove lru entry
-    if (lruKey) {
-      cache.delete(lruKey);
-    }
   }
 
   return {
     get(key) {
       const entry = cache.get(key);
+      console.log("Size", currSize, cache.size)
       if (entry) {
         entry.last = performance.now();
         return entry.value;
       }
     },
     set(key, value) {
-      cache.set(key, { last: performance.now(), value });
-      if (cache.size > max) requestIdle(evict);
+      let setValue = { 
+        last: performance.now(), 
+        size: new Blob([value]).size,
+        value 
+      };
+
+      if (cache.has(key)) {
+        currSize -= cache.get(key).size;
+      }
+      cache.set(key, setValue);
+      currSize += setValue.size;
+
+      if (currSize > max) requestIdle(evict);
       return value;
     },
     clear() { cache = new Map; }

packages/duckdb/src/Cache.js

@@ -22,14 +22,15 @@ class CacheEntry {
 
 export class Cache {
   constructor({
-    max = 10000, // max entries
+    max = 10 * 1024 * 1024, // 10 MB cache size as default
     dir = DEFAULT_CACHE_DIR,
     ttl = DEFAULT_TTL
   }) {
     this.cache = new Map;
     this.max = max;
     this.dir = dir;
     this.ttl = ttl;
+    this.curr_size = 0;
     readEntries(dir, this.cache);
   }
 
@@ -52,40 +53,46 @@ export class Cache {
   set(key, data, { persist = false, ttl = this.ttl } = {}) {
     const entry = new CacheEntry(data, persist ? Infinity : ttl);
     this.cache.set(key, entry);
+    this.curr_size += new Blob([entry]).size;
     if (persist) writeEntry(this.dir, key, entry);
     if (this.shouldEvict()) setTimeout(() => this.evict());
     return this;
   }
 
   shouldEvict() {
-    return this.cache.size > this.max;
+    return this.curr_size > this.max;
   }
 
   evict() {
     const expire = performance.now();
-    let lruKey = null;
-    let lruLast = Infinity;
 
-    for (const [key, entry] of this.cache) {
-      const { last } = entry;
-      if (last === Infinity) continue;
-
-      // least recently used entry seen so far
-      if (last < lruLast) {
-        lruKey = key;
-        lruLast = last;
+    while (this.shouldEvict()) {
+      let lruKey = null;
+      let lruLast = Infinity;
+
+      for (const [key, entry] of this.cache) {
+        const { last } = entry;
+        if (last === Infinity) continue;
+
+        // least recently used entry seen so far
+        if (last < lruLast) {
+          lruKey = key;
+          lruLast = last;
+        }
+
+        // remove if time since last access exceeds ttl
+        if (expire > last) {
+          this.cache.delete(key);
+          this.curr_size -= new Blob([entry]).size;
+        }
       }
 
-      // remove if time since last access exceeds ttl
-      if (expire > last) {
-        this.cache.delete(key);
+      // remove lru entry
+      if (this.shouldEvict() && this.cache.has(lruKey)) {
+        this.curr_size -= new Blob([this.cache.get(lruKey)]).size;
+        this.cache.delete(lruKey);
       }
     }
-
-    // remove lru entry
-    if (this.cache.size > this.max && lruKey) {
-      this.cache.delete(lruKey);
-    }
   }
 }