2026-1-6

venv/Lib/site-packages/whoosh/filedb/gae.py

@@ -0,0 +1,164 @@
+"""
+This module contains EXPERIMENTAL support for storing a Whoosh index's files in
+the Google App Engine blobstore. This will use a lot of RAM since all files are
+loaded into RAM, but it potentially useful as a workaround for the lack of file
+storage in Google App Engine.
+
+Use at your own risk, but please report any problems to me so I can fix them.
+
+To create a new index::
+
+    from whoosh.filedb.gae import DatastoreStorage
+
+    ix = DatastoreStorage().create_index(schema)
+
+To open an existing index::
+
+    ix = DatastoreStorage().open_index()
+"""
+
+import time
+
+from google.appengine.api import memcache  # @UnresolvedImport
+from google.appengine.ext import db  # @UnresolvedImport
+
+from whoosh.compat import BytesIO
+from whoosh.index import TOC, FileIndex, _DEF_INDEX_NAME
+from whoosh.filedb.filestore import ReadOnlyError, Storage
+from whoosh.filedb.structfile import StructFile
+
+
+class DatastoreFile(db.Model):
+    """A file-like object that is backed by a BytesIO() object whose contents
+    is loaded from a BlobProperty in the app engine datastore.
+    """
+
+    value = db.BlobProperty()
+    mtime = db.IntegerProperty(default=0)
+
+    def __init__(self, *args, **kwargs):
+        super(DatastoreFile, self).__init__(*args, **kwargs)
+        self.data = BytesIO()
+
+    @classmethod
+    def loadfile(cls, name):
+        value = memcache.get(name, namespace="DatastoreFile")
+        if value is None:
+            file = cls.get_by_key_name(name)
+            memcache.set(name, file.value, namespace="DatastoreFile")
+        else:
+            file = cls(value=value)
+        file.data = BytesIO(file.value)
+        return file
+
+    def close(self):
+        oldvalue = self.value
+        self.value = self.getvalue()
+        if oldvalue != self.value:
+            self.mtime = int(time.time())
+            self.put()
+            memcache.set(self.key().id_or_name(), self.value,
+                         namespace="DatastoreFile")
+
+    def tell(self):
+        return self.data.tell()
+
+    def write(self, data):
+        return self.data.write(data)
+
+    def read(self, length):
+        return self.data.read(length)
+
+    def seek(self, *args):
+        return self.data.seek(*args)
+
+    def readline(self):
+        return self.data.readline()
+
+    def getvalue(self):
+        return self.data.getvalue()
+
+
+class MemcacheLock(object):
+    def __init__(self, name):
+        self.name = name
+
+    def acquire(self, blocking=False):
+        val = memcache.add(self.name, "L", 360, namespace="whooshlocks")
+
+        if blocking and not val:
+            # Simulate blocking by retrying the acquire over and over
+            import time
+            while not val:
+                time.sleep(0.1)
+                val = memcache.add(self.name, "", 360, namespace="whooshlocks")
+
+        return val
+
+    def release(self):
+        memcache.delete(self.name, namespace="whooshlocks")
+
+
+class DatastoreStorage(Storage):
+    """An implementation of :class:`whoosh.store.Storage` that stores files in
+    the app engine datastore as blob properties.
+    """
+
+    def create_index(self, schema, indexname=_DEF_INDEX_NAME):
+        if self.readonly:
+            raise ReadOnlyError
+
+        TOC.create(self, schema, indexname)
+        return FileIndex(self, schema, indexname)
+
+    def open_index(self, indexname=_DEF_INDEX_NAME, schema=None):
+        return FileIndex(self, schema=schema, indexname=indexname)
+
+    def list(self):
+        query = DatastoreFile.all()
+        keys = []
+        for file in query:
+            keys.append(file.key().id_or_name())
+        return keys
+
+    def clean(self):
+        pass
+
+    def total_size(self):
+        return sum(self.file_length(f) for f in self.list())
+
+    def file_exists(self, name):
+        return DatastoreFile.get_by_key_name(name) is not None
+
+    def file_modified(self, name):
+        return DatastoreFile.get_by_key_name(name).mtime
+
+    def file_length(self, name):
+        return len(DatastoreFile.get_by_key_name(name).value)
+
+    def delete_file(self, name):
+        memcache.delete(name, namespace="DatastoreFile")
+        return DatastoreFile.get_by_key_name(name).delete()
+
+    def rename_file(self, name, newname, safe=False):
+        file = DatastoreFile.get_by_key_name(name)
+        newfile = DatastoreFile(key_name=newname)
+        newfile.value = file.value
+        newfile.mtime = file.mtime
+        newfile.put()
+        file.delete()
+
+    def create_file(self, name, **kwargs):
+        f = StructFile(DatastoreFile(key_name=name), name=name,
+                       onclose=lambda sfile: sfile.file.close())
+        return f
+
+    def open_file(self, name, *args, **kwargs):
+        return StructFile(DatastoreFile.loadfile(name))
+
+    def lock(self, name):
+        return MemcacheLock(name)
+
+    def temp_storage(self, name=None):
+        tempstore = DatastoreStorage()
+        return tempstore.create()