wehub-resource-sync
76 lines
2.4 KiB
Python
76 lines
2.4 KiB
Python
import logging
|
|
from warnings import warn
|
|
|
|
import pathway as pw
|
|
from dotenv import load_dotenv
|
|
from pathway.xpacks.llm.document_store import DocumentStore
|
|
from pathway.xpacks.llm.servers import DocumentStoreServer
|
|
from pydantic import BaseModel, ConfigDict, InstanceOf
|
|
|
|
# To use advanced features with Pathway Live Data Framework Scale, get your free license key from
|
|
# https://pathway.com/features and paste it below.
|
|
# To use Pathway Live Data Framework Community, comment out the line below.
|
|
pw.set_license_key("demo-license-key-with-telemetry")
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(name)s %(levelname)s %(message)s",
|
|
datefmt="%Y-%m-%d %H:%M:%S",
|
|
)
|
|
|
|
load_dotenv()
|
|
|
|
|
|
class App(BaseModel):
|
|
document_store: InstanceOf[DocumentStore]
|
|
host: str = "0.0.0.0"
|
|
port: int = 8000
|
|
|
|
with_cache: bool | None = None # deprecated
|
|
persistence_backend: pw.persistence.Backend | None = None
|
|
persistence_mode: pw.PersistenceMode | None = pw.PersistenceMode.UDF_CACHING
|
|
terminate_on_error: bool = False
|
|
|
|
def run(self) -> None:
|
|
server = DocumentStoreServer( # noqa: F841
|
|
self.host, self.port, self.document_store
|
|
)
|
|
if self.persistence_mode is None:
|
|
if self.with_cache is True:
|
|
warn(
|
|
"`with_cache` is deprecated. Please use `persistence_mode` instead.",
|
|
DeprecationWarning,
|
|
)
|
|
persistence_mode = pw.PersistenceMode.UDF_CACHING
|
|
else:
|
|
persistence_mode = None
|
|
else:
|
|
persistence_mode = self.persistence_mode
|
|
|
|
if persistence_mode is not None:
|
|
if self.persistence_backend is None:
|
|
persistence_backend = pw.persistence.Backend.filesystem("./Cache")
|
|
else:
|
|
persistence_backend = self.persistence_backend
|
|
persistence_config = pw.persistence.Config(
|
|
persistence_backend,
|
|
persistence_mode=persistence_mode,
|
|
)
|
|
else:
|
|
persistence_config = None
|
|
|
|
pw.run(
|
|
persistence_config=persistence_config,
|
|
terminate_on_error=self.terminate_on_error,
|
|
monitoring_level=pw.MonitoringLevel.NONE,
|
|
)
|
|
|
|
model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
with open("app.yaml") as f:
|
|
config = pw.load_yaml(f)
|
|
app = App(**config)
|
|
app.run()
|