chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
# This file specifies files that are *not* uploaded to Google Cloud
|
||||
# using gcloud. It follows the same syntax as .gitignore, with the addition of
|
||||
# "#!include" directives (which insert the entries of the given .gitignore-style
|
||||
# file at that point).
|
||||
#
|
||||
# For more information, run:
|
||||
# $ gcloud topic gcloudignore
|
||||
#
|
||||
.gcloudignore
|
||||
# If you would like to upload your .git directory, .gitignore file or files
|
||||
# from your .gitignore file, remove the corresponding line
|
||||
# below:
|
||||
.git
|
||||
.gitignore
|
||||
|
||||
# Python pycache:
|
||||
__pycache__/
|
||||
# Ignored by the build system
|
||||
/setup.cfg
|
||||
my_env
|
||||
@@ -0,0 +1,7 @@
|
||||
__pycache__/
|
||||
.idea/
|
||||
.vscode/
|
||||
.venv/
|
||||
env
|
||||
bin
|
||||
lib
|
||||
@@ -0,0 +1,28 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
FROM python:3.11-alpine
|
||||
|
||||
COPY requirements.txt /tmp
|
||||
RUN pip install -r /tmp/requirements.txt
|
||||
|
||||
WORKDIR /app
|
||||
COPY . /app/
|
||||
|
||||
ENV ENVIRONMENT="development"
|
||||
ENV BIG_QUERY_DATASET=""
|
||||
ENV FRONTEND_URL=""
|
||||
|
||||
EXPOSE 8080
|
||||
ENTRYPOINT ["gunicorn", "main:app", "--workers=4", "--worker-class=uvicorn.workers.UvicornWorker", "--timeout=36000", "--bind=0.0.0.0:8080"]
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# syntax=docker/dockerfile:1.4
|
||||
FROM python:3.11 AS builder
|
||||
|
||||
# Build arguments
|
||||
ARG IS_FIRST_DEPLOYMENT="False"
|
||||
ARG GCLOUD_PROJECT=""
|
||||
ARG ENVIRONMENT="development"
|
||||
ARG FRONTEND_URL=""
|
||||
ARG BIG_QUERY_DATASET=""
|
||||
ARG VERTEX_AI_LOCATION=""
|
||||
ARG VERTEX_AI_DATASTORE_ID=""
|
||||
ARG VERTEX_AI_ENGINE_ID=""
|
||||
|
||||
# Environment variables for the build process
|
||||
ENV IS_FIRST_DEPLOYMENT=$IS_FIRST_DEPLOYMENT
|
||||
ENV GCLOUD_PROJECT=$GCLOUD_PROJECT
|
||||
ENV ENVIRONMENT=$ENVIRONMENT
|
||||
ENV FRONTEND_URL=$FRONTEND_URL
|
||||
ENV BIG_QUERY_DATASET=$BIG_QUERY_DATASET
|
||||
ENV VERTEX_AI_LOCATION=$VERTEX_AI_LOCATION
|
||||
ENV VERTEX_AI_DATASTORE_ID=$VERTEX_AI_DATASTORE_ID
|
||||
ENV VERTEX_AI_ENGINE_ID=$VERTEX_AI_ENGINE_ID
|
||||
|
||||
WORKDIR /code
|
||||
|
||||
COPY . .
|
||||
COPY setup.py ./setup.py
|
||||
RUN pip install --no-cache-dir -r /code/requirements.txt
|
||||
|
||||
RUN --mount=type=secret,id=gcp_credentials,target=/tmp/gcp_adc.json,required=true \
|
||||
if [ "$IS_FIRST_DEPLOYMENT" = "True" ]; then \
|
||||
echo "--- Running first deployment setup (IS_FIRST_DEPLOYMENT=True) ---"; \
|
||||
echo "Using GCLOUD_PROJECT=${GCLOUD_PROJECT} for setup"; \
|
||||
\
|
||||
if [ ! -f /tmp/gcp_adc.json ]; then \
|
||||
echo "CRITICAL ERROR: GCP credentials secret was expected but not mounted to /tmp/gcp_adc.json" >&2; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
echo "DEBUG: GCP credentials secret successfully mounted to /tmp/gcp_adc.json"; \
|
||||
\
|
||||
# Set GOOGLE_APPLICATION_CREDENTIALS specifically for this RUN command's execution context
|
||||
export GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp_adc.json; \
|
||||
echo "DEBUG: GOOGLE_APPLICATION_CREDENTIALS for setup is set to: $GOOGLE_APPLICATION_CREDENTIALS"; \
|
||||
\
|
||||
echo "Executing setup.py ..."; \
|
||||
# Now setup.py can make requests to GCP with provided credentials
|
||||
python /code/setup.py || { echo "setup.py with failed!"; exit 1; }; \
|
||||
echo "--- First deployment setup finished ---"; \
|
||||
else \
|
||||
echo "--- IS_FIRST_DEPLOYMENT is False, skipping first deployment setup. ---"; \
|
||||
fi
|
||||
|
||||
|
||||
FROM python:3.11-slim AS final
|
||||
|
||||
ARG GCLOUD_PROJECT=""
|
||||
ARG GOOGLE_APPLICATION_CREDENTIALS=""
|
||||
ARG ENVIRONMENT="development"
|
||||
ARG FRONTEND_URL=""
|
||||
ARG BIG_QUERY_DATASET=""
|
||||
|
||||
# Environment variables for the build process
|
||||
ENV GCLOUD_PROJECT=$GCLOUD_PROJECT
|
||||
ENV GOOGLE_APPLICATION_CREDENTIALS="/root/.config/gcloud/application_default_credentials.json"
|
||||
ENV ENVIRONMENT=$ENVIRONMENT
|
||||
ENV FRONTEND_URL=$FRONTEND_URL
|
||||
ENV BIG_QUERY_DATASET=$BIG_QUERY_DATASET
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /code/requirements.txt /app/requirements.txt
|
||||
RUN pip install --no-cache-dir -r /app/requirements.txt
|
||||
|
||||
COPY --from=builder /code/main.py /app/main.py
|
||||
COPY --from=builder /code/src /app/src
|
||||
|
||||
EXPOSE 8080
|
||||
ENTRYPOINT ["gunicorn", "main:app", "--workers=4", "--worker-class=uvicorn.workers.UvicornWorker", "--timeout=36000", "--bind=0.0.0.0:8080"]
|
||||
@@ -0,0 +1,132 @@
|
||||
# QuickBotApp document-search-using-agent-builder
|
||||
|
||||
## Setting up
|
||||
### 1. Create virtualenv and install dependencies
|
||||
Create a virtual environment on the root of the application, activate it and install the requirements
|
||||
```
|
||||
# check if you are already in the env
|
||||
pip -V
|
||||
|
||||
# if not then
|
||||
python3 -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip3 install -r requirements.txt
|
||||
```
|
||||
|
||||
> **IMPORTANT!** VS Code may not recognize your env, in that case type "ctrl + shift + P", then select "Python: Select Interpreter" and then select "Enter interpreter path..." and then select your .venv python interpreter, in this case .backend/.venv/bin/python
|
||||
|
||||
|
||||
### 2. Setup gcloud credentials
|
||||
```
|
||||
gcloud auth list
|
||||
gcloud config list
|
||||
|
||||
gcloud auth login
|
||||
gcloud config set project <your project id>
|
||||
gcloud auth application-default set-quota-project <your project id>
|
||||
|
||||
gcloud auth list
|
||||
gcloud config list
|
||||
```
|
||||
|
||||
### 3. Add environment variables
|
||||
|
||||
#### If you have Mac or Windows (or if you are using zsh console on Linux)
|
||||
```
|
||||
. ./local.env
|
||||
```
|
||||
|
||||
#### If you have Linux
|
||||
Open the file .venv/bin/activate and paste the env variables from `.local.env` after the PATH export, like this:
|
||||
```
|
||||
...
|
||||
|
||||
_OLD_VIRTUAL_PATH="$PATH"
|
||||
PATH="$VIRTUAL_ENV/bin:$PATH"
|
||||
export PATH
|
||||
|
||||
# Quickbot env variables
|
||||
export ENVIRONMENT=development
|
||||
export FRONTEND_URL=http://localhost:4200
|
||||
export BIG_QUERY_DATASET=eren
|
||||
...
|
||||
```
|
||||
|
||||
Check that the env variables has been taken into account, running:
|
||||
```
|
||||
env
|
||||
```
|
||||
You should see the new env variables set there
|
||||
|
||||
|
||||
|
||||
### 4. Running the set up script
|
||||
```
|
||||
python3 setup.py
|
||||
```
|
||||
|
||||
### 5. Run the application
|
||||
Finally run using uvicorn
|
||||
```
|
||||
uvicorn main:app --reload --port 8080
|
||||
```
|
||||
|
||||
## Code Styling & Commit Guidelines
|
||||
|
||||
To maintain code quality and consistency:
|
||||
|
||||
* **TypeScript (Frontend):** We follow [Angular Coding Style Guide](https://angular.dev/style-guide) by leveraging the use of [Google's TypeScript Style Guide](https://github.com/google/gts) using `gts`. This includes a formatter, linter, and automatic code fixer.
|
||||
* **Python (Backend):** We adhere to the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html), using tools like `pylint` and `black` for linting and formatting.
|
||||
* **Commit Messages:** We suggest following [Angular's Commit Message Guidelines](https://github.com/angular/angular/blob/main/contributing-docs/commit-message-guidelines.md) to create clear and descriptive commit messages.
|
||||
|
||||
#### Frontend (TypeScript with `gts`)
|
||||
|
||||
1. **Initialize `gts` (if not already done in the project):**
|
||||
Navigate to the `frontend/` directory and run:
|
||||
```bash
|
||||
npx gts init
|
||||
```
|
||||
This will set up `gts` and create necessary configuration files (like `tsconfig.json`). Ensure your `tsconfig.json` (or a related gts config file like `.gtsrc`) includes an extension for `gts` defaults, typically:
|
||||
```json
|
||||
{
|
||||
"extends": "./node_modules/gts/tsconfig-google.json",
|
||||
// ... other configurations
|
||||
}
|
||||
```
|
||||
2. **Check for linting issues:**
|
||||
```bash
|
||||
npm run lint
|
||||
```
|
||||
(This assumes a `lint` script is defined in `package.json`, e.g., `"lint": "gts lint"`)
|
||||
3. **Fix linting issues automatically (where possible):**
|
||||
```bash
|
||||
npm run fix
|
||||
```
|
||||
(This assumes a `fix` script is defined in `package.json`, e.g., `"fix": "gts fix"`)
|
||||
|
||||
#### Backend (Python with `pylint` and `black`)
|
||||
|
||||
1. **Ensure Dependencies are Installed:**
|
||||
Add `pylint` and `black` to your `backend/requirements.txt` file:
|
||||
```
|
||||
pylint
|
||||
black
|
||||
```
|
||||
Then install them within your virtual environment:
|
||||
```bash
|
||||
pip install pylint black
|
||||
# or pip install -r requirements.txt
|
||||
```
|
||||
2. **Configure `pylint`:**
|
||||
It's recommended to have a `.pylintrc` file in your `backend/` directory to configure `pylint` rules. You might need to copy a standard one or generate one (`pylint --generate-rcfile > .pylintrc`).
|
||||
3. **Check for linting issues with `pylint`:**
|
||||
Navigate to the `backend/` directory and run:
|
||||
```bash
|
||||
pylint .
|
||||
```
|
||||
(Or specify modules/packages: `pylint your_module_name`)
|
||||
4. **Format code with `black`:**
|
||||
To automatically format all Python files in the current directory and subdirectories:
|
||||
```bash
|
||||
python -m black . --line-length=80
|
||||
```
|
||||
@@ -0,0 +1,14 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
export ENVIRONMENT=development
|
||||
export FRONTEND_URL=http://localhost:4200
|
||||
export BIG_QUERY_DATASET=eren
|
||||
@@ -0,0 +1,104 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Main FastAPI application entry point for the Quick Bot Backend.
|
||||
|
||||
This module initializes the FastAPI application, configures CORS based on
|
||||
the environment, defines root and version endpoints, includes API routers
|
||||
(e.g., for search functionality), and provides an endpoint for audio
|
||||
transcription using Google Cloud Speech-to-Text.
|
||||
"""
|
||||
|
||||
from fastapi import FastAPI, File, UploadFile
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from src.controller.search import router as search_router
|
||||
from google.cloud import speech
|
||||
from os import getenv
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
|
||||
def configure_cors(app):
|
||||
"""Configures CORS middleware based on the environment."""
|
||||
environment = getenv("ENVIRONMENT")
|
||||
allowed_origins = []
|
||||
|
||||
if environment == "production":
|
||||
frontend_url = getenv("FRONTEND_URL")
|
||||
if not frontend_url:
|
||||
raise ValueError(
|
||||
"FRONTEND_URL environment variable not set in production"
|
||||
)
|
||||
allowed_origins.append(frontend_url)
|
||||
elif environment == "development":
|
||||
allowed_origins.append("*") # Allow all origins in development
|
||||
else:
|
||||
raise ValueError(
|
||||
f"""Invalid ENVIRONMENT: {environment}.
|
||||
Must be 'production' or 'development'"""
|
||||
)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=allowed_origins,
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
# Create a route to handle GET requests on root
|
||||
@app.get("/")
|
||||
async def root():
|
||||
return "You are calling Quick Bot Backend"
|
||||
|
||||
|
||||
# Create a route to handle GET requests on /version
|
||||
@app.get("/api/version")
|
||||
def version():
|
||||
return "v0.0.1"
|
||||
|
||||
|
||||
@app.post("/api/audio_chat")
|
||||
async def audio_chat(audio_file: UploadFile = File(...)):
|
||||
client = speech.SpeechClient()
|
||||
audio_content = await audio_file.read()
|
||||
audio = speech.RecognitionAudio(content=audio_content)
|
||||
config = speech.RecognitionConfig(
|
||||
language_code="en-US",
|
||||
sample_rate_hertz=48000,
|
||||
model="default",
|
||||
audio_channel_count=1,
|
||||
enable_word_confidence=True,
|
||||
enable_word_time_offsets=True,
|
||||
)
|
||||
|
||||
operation = client.long_running_recognize(config=config, audio=audio)
|
||||
|
||||
print("Waiting for operation to complete...")
|
||||
response = operation.result(timeout=90)
|
||||
print(response)
|
||||
|
||||
text = ""
|
||||
for result in response.results:
|
||||
print(f"Transcript: {result.alternatives[0].transcript}")
|
||||
text = result.alternatives[0].transcript
|
||||
|
||||
return text, 200
|
||||
|
||||
|
||||
configure_cors(app)
|
||||
|
||||
app.include_router(search_router)
|
||||
@@ -0,0 +1,399 @@
|
||||
# This Pylint rcfile contains a best-effort configuration to uphold the
|
||||
# best-practices and style described in the Google Python style guide:
|
||||
# https://google.github.io/styleguide/pyguide.html
|
||||
#
|
||||
# Its canonical open-source location is:
|
||||
# https://google.github.io/styleguide/pylintrc
|
||||
|
||||
[MAIN]
|
||||
|
||||
# Files or directories to be skipped. They should be base names, not paths.
|
||||
ignore=third_party
|
||||
|
||||
# Files or directories matching the regex patterns are skipped. The regex
|
||||
# matches against base names, not paths.
|
||||
ignore-patterns=
|
||||
|
||||
# Pickle collected data for later comparisons.
|
||||
persistent=no
|
||||
|
||||
# List of plugins (as comma separated values of python modules names) to load,
|
||||
# usually to register additional checkers.
|
||||
load-plugins=
|
||||
|
||||
# Use multiple processes to speed up Pylint.
|
||||
jobs=4
|
||||
|
||||
# Allow loading of arbitrary C extensions. Extensions are imported into the
|
||||
# active Python interpreter and may run arbitrary code.
|
||||
unsafe-load-any-extension=no
|
||||
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
|
||||
# Only show warnings with the listed confidence levels. Leave empty to show
|
||||
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
|
||||
confidence=
|
||||
|
||||
# Enable the message, report, category or checker with the given id(s). You can
|
||||
# either give multiple identifier separated by comma (,) or put this option
|
||||
# multiple time (only on the command line, not in the configuration file where
|
||||
# it should appear only once). See also the "--disable" option for examples.
|
||||
#enable=
|
||||
|
||||
# Disable the message, report, category or checker with the given id(s). You
|
||||
# can either give multiple identifiers separated by comma (,) or put this
|
||||
# option multiple times (only on the command line, not in the configuration
|
||||
# file where it should appear only once).You can also use "--disable=all" to
|
||||
# disable everything first and then reenable specific checks. For example, if
|
||||
# you want to run only the similarities checker, you can use "--disable=all
|
||||
# --enable=similarities". If you want to run only the classes checker, but have
|
||||
# no Warning level messages displayed, use"--disable=all --enable=classes
|
||||
# --disable=W"
|
||||
disable=R,
|
||||
abstract-method,
|
||||
apply-builtin,
|
||||
arguments-differ,
|
||||
attribute-defined-outside-init,
|
||||
backtick,
|
||||
bad-option-value,
|
||||
basestring-builtin,
|
||||
buffer-builtin,
|
||||
c-extension-no-member,
|
||||
consider-using-enumerate,
|
||||
cmp-builtin,
|
||||
cmp-method,
|
||||
coerce-builtin,
|
||||
coerce-method,
|
||||
delslice-method,
|
||||
div-method,
|
||||
eq-without-hash,
|
||||
execfile-builtin,
|
||||
file-builtin,
|
||||
filter-builtin-not-iterating,
|
||||
fixme,
|
||||
getslice-method,
|
||||
global-statement,
|
||||
hex-method,
|
||||
idiv-method,
|
||||
implicit-str-concat,
|
||||
import-error,
|
||||
import-self,
|
||||
import-star-module-level,
|
||||
input-builtin,
|
||||
intern-builtin,
|
||||
invalid-str-codec,
|
||||
locally-disabled,
|
||||
long-builtin,
|
||||
long-suffix,
|
||||
map-builtin-not-iterating,
|
||||
misplaced-comparison-constant,
|
||||
missing-function-docstring,
|
||||
metaclass-assignment,
|
||||
next-method-called,
|
||||
next-method-defined,
|
||||
no-absolute-import,
|
||||
no-init, # added
|
||||
no-member,
|
||||
no-name-in-module,
|
||||
no-self-use,
|
||||
nonzero-method,
|
||||
oct-method,
|
||||
old-division,
|
||||
old-ne-operator,
|
||||
old-octal-literal,
|
||||
old-raise-syntax,
|
||||
parameter-unpacking,
|
||||
print-statement,
|
||||
raising-string,
|
||||
range-builtin-not-iterating,
|
||||
raw_input-builtin,
|
||||
rdiv-method,
|
||||
reduce-builtin,
|
||||
relative-import,
|
||||
reload-builtin,
|
||||
round-builtin,
|
||||
setslice-method,
|
||||
signature-differs,
|
||||
standarderror-builtin,
|
||||
suppressed-message,
|
||||
sys-max-int,
|
||||
trailing-newlines,
|
||||
unichr-builtin,
|
||||
unicode-builtin,
|
||||
unnecessary-pass,
|
||||
unpacking-in-except,
|
||||
useless-else-on-loop,
|
||||
useless-suppression,
|
||||
using-cmp-argument,
|
||||
wrong-import-order,
|
||||
xrange-builtin,
|
||||
zip-builtin-not-iterating,
|
||||
|
||||
|
||||
[REPORTS]
|
||||
|
||||
# Set the output format. Available formats are text, parseable, colorized, msvs
|
||||
# (visual studio) and html. You can also give a reporter class, eg
|
||||
# mypackage.mymodule.MyReporterClass.
|
||||
output-format=text
|
||||
|
||||
# Tells whether to display a full report or only the messages
|
||||
reports=no
|
||||
|
||||
# Python expression which should return a note less than 10 (10 is the highest
|
||||
# note). You have access to the variables errors warning, statement which
|
||||
# respectively contain the number of errors / warnings messages and the total
|
||||
# number of statements analyzed. This is used by the global evaluation report
|
||||
# (RP0004).
|
||||
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
|
||||
|
||||
# Template used to display messages. This is a python new-style format string
|
||||
# used to format the message information. See doc for all details
|
||||
#msg-template=
|
||||
|
||||
|
||||
[BASIC]
|
||||
|
||||
# Good variable names which should always be accepted, separated by a comma
|
||||
good-names=main,_
|
||||
|
||||
# Bad variable names which should always be refused, separated by a comma
|
||||
bad-names=
|
||||
|
||||
# Colon-delimited sets of names that determine each other's naming style when
|
||||
# the name regexes allow several styles.
|
||||
name-group=
|
||||
|
||||
# Include a hint for the correct naming format with invalid-name
|
||||
include-naming-hint=no
|
||||
|
||||
# List of decorators that produce properties, such as abc.abstractproperty. Add
|
||||
# to this list to register other decorators that produce valid properties.
|
||||
property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl
|
||||
|
||||
# Regular expression matching correct function names
|
||||
function-rgx=^(?:(?P<exempt>setUp|tearDown|setUpModule|tearDownModule)|(?P<camel_case>_?[A-Z][a-zA-Z0-9]*)|(?P<snake_case>_?[a-z][a-z0-9_]*))$
|
||||
|
||||
# Regular expression matching correct variable names
|
||||
variable-rgx=^[a-z][a-z0-9_]*$
|
||||
|
||||
# Regular expression matching correct constant names
|
||||
const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
|
||||
|
||||
# Regular expression matching correct attribute names
|
||||
attr-rgx=^_{0,2}[a-z][a-z0-9_]*$
|
||||
|
||||
# Regular expression matching correct argument names
|
||||
argument-rgx=^[a-z][a-z0-9_]*$
|
||||
|
||||
# Regular expression matching correct class attribute names
|
||||
class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
|
||||
|
||||
# Regular expression matching correct inline iteration names
|
||||
inlinevar-rgx=^[a-z][a-z0-9_]*$
|
||||
|
||||
# Regular expression matching correct class names
|
||||
class-rgx=^_?[A-Z][a-zA-Z0-9]*$
|
||||
|
||||
# Regular expression matching correct module names
|
||||
module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$
|
||||
|
||||
# Regular expression matching correct method names
|
||||
method-rgx=(?x)^(?:(?P<exempt>_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P<camel_case>_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P<snake_case>_{0,2}[a-z][a-z0-9_]*))$
|
||||
|
||||
# Regular expression which should only match function or class names that do
|
||||
# not require a docstring.
|
||||
no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$
|
||||
|
||||
# Minimum line length for functions/classes that require docstrings, shorter
|
||||
# ones are exempt.
|
||||
docstring-min-length=12
|
||||
|
||||
|
||||
[TYPECHECK]
|
||||
|
||||
# List of decorators that produce context managers, such as
|
||||
# contextlib.contextmanager. Add to this list to register other decorators that
|
||||
# produce valid context managers.
|
||||
contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager
|
||||
|
||||
# List of module names for which member attributes should not be checked
|
||||
# (useful for modules/projects where namespaces are manipulated during runtime
|
||||
# and thus existing member attributes cannot be deduced by static analysis. It
|
||||
# supports qualified module names, as well as Unix pattern matching.
|
||||
ignored-modules=
|
||||
|
||||
# List of class names for which member attributes should not be checked (useful
|
||||
# for classes with dynamically set attributes). This supports the use of
|
||||
# qualified names.
|
||||
ignored-classes=optparse.Values,thread._local,_thread._local
|
||||
|
||||
# List of members which are set dynamically and missed by pylint inference
|
||||
# system, and so shouldn't trigger E1101 when accessed. Python regular
|
||||
# expressions are accepted.
|
||||
generated-members=
|
||||
|
||||
|
||||
[FORMAT]
|
||||
|
||||
# Maximum number of characters on a single line.
|
||||
max-line-length=80
|
||||
|
||||
# TODO(https://github.com/pylint-dev/pylint/issues/3352): Direct pylint to exempt
|
||||
# lines made too long by directives to pytype.
|
||||
|
||||
# Regexp for a line that is allowed to be longer than the limit.
|
||||
ignore-long-lines=(?x)(
|
||||
^\s*(\#\ )?<?https?://\S+>?$|
|
||||
^\s*(from\s+\S+\s+)?import\s+.+$)
|
||||
|
||||
# Allow the body of an if to be on the same line as the test if there is no
|
||||
# else.
|
||||
single-line-if-stmt=yes
|
||||
|
||||
# Maximum number of lines in a module
|
||||
max-module-lines=99999
|
||||
|
||||
# String used as indentation unit. The internal Google style guide mandates 2
|
||||
# spaces. Google's externaly-published style guide says 4, consistent with
|
||||
# PEP 8. Here, we use 2 spaces, for conformity with many open-sourced Google
|
||||
# projects (like TensorFlow).
|
||||
indent-string=' '
|
||||
|
||||
# Number of spaces of indent required inside a hanging or continued line.
|
||||
indent-after-paren=4
|
||||
|
||||
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
|
||||
expected-line-ending-format=
|
||||
|
||||
|
||||
[MISCELLANEOUS]
|
||||
|
||||
# List of note tags to take in consideration, separated by a comma.
|
||||
notes=TODO
|
||||
|
||||
|
||||
[STRING]
|
||||
|
||||
# This flag controls whether inconsistent-quotes generates a warning when the
|
||||
# character used as a quote delimiter is used inconsistently within a module.
|
||||
check-quote-consistency=yes
|
||||
|
||||
|
||||
[VARIABLES]
|
||||
|
||||
# Tells whether we should check for unused import in __init__ files.
|
||||
init-import=no
|
||||
|
||||
# A regular expression matching the name of dummy variables (i.e. expectedly
|
||||
# not used).
|
||||
dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_)
|
||||
|
||||
# List of additional names supposed to be defined in builtins. Remember that
|
||||
# you should avoid to define new builtins when possible.
|
||||
additional-builtins=
|
||||
|
||||
# List of strings which can identify a callback function by name. A callback
|
||||
# name must start or end with one of those strings.
|
||||
callbacks=cb_,_cb
|
||||
|
||||
# List of qualified module names which can have objects that can redefine
|
||||
# builtins.
|
||||
redefining-builtins-modules=six,six.moves,past.builtins,future.builtins,functools
|
||||
|
||||
|
||||
[LOGGING]
|
||||
|
||||
# Logging modules to check that the string format arguments are in logging
|
||||
# function parameter format
|
||||
logging-modules=logging,absl.logging,tensorflow.io.logging
|
||||
|
||||
|
||||
[SIMILARITIES]
|
||||
|
||||
# Minimum lines number of a similarity.
|
||||
min-similarity-lines=4
|
||||
|
||||
# Ignore comments when computing similarities.
|
||||
ignore-comments=yes
|
||||
|
||||
# Ignore docstrings when computing similarities.
|
||||
ignore-docstrings=yes
|
||||
|
||||
# Ignore imports when computing similarities.
|
||||
ignore-imports=no
|
||||
|
||||
|
||||
[SPELLING]
|
||||
|
||||
# Spelling dictionary name. Available dictionaries: none. To make it working
|
||||
# install python-enchant package.
|
||||
spelling-dict=
|
||||
|
||||
# List of comma separated words that should not be checked.
|
||||
spelling-ignore-words=
|
||||
|
||||
# A path to a file that contains private dictionary; one word per line.
|
||||
spelling-private-dict-file=
|
||||
|
||||
# Tells whether to store unknown words to indicated private dictionary in
|
||||
# --spelling-private-dict-file option instead of raising a message.
|
||||
spelling-store-unknown-words=no
|
||||
|
||||
|
||||
[IMPORTS]
|
||||
|
||||
# Deprecated modules which should not be used, separated by a comma
|
||||
deprecated-modules=regsub,
|
||||
TERMIOS,
|
||||
Bastion,
|
||||
rexec,
|
||||
sets
|
||||
|
||||
# Create a graph of every (i.e. internal and external) dependencies in the
|
||||
# given file (report RP0402 must not be disabled)
|
||||
import-graph=
|
||||
|
||||
# Create a graph of external dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
ext-import-graph=
|
||||
|
||||
# Create a graph of internal dependencies in the given file (report RP0402 must
|
||||
# not be disabled)
|
||||
int-import-graph=
|
||||
|
||||
# Force import order to recognize a module as part of the standard
|
||||
# compatibility libraries.
|
||||
known-standard-library=
|
||||
|
||||
# Force import order to recognize a module as part of a third party library.
|
||||
known-third-party=enchant, absl
|
||||
|
||||
# Analyse import fallback blocks. This can be used to support both Python 2 and
|
||||
# 3 compatible code, which means that the block might have code that exists
|
||||
# only in one or another interpreter, leading to false positives when analysed.
|
||||
analyse-fallback-blocks=no
|
||||
|
||||
|
||||
[CLASSES]
|
||||
|
||||
# List of method names used to declare (i.e. assign) instance attributes.
|
||||
defining-attr-methods=__init__,
|
||||
__new__,
|
||||
setUp
|
||||
|
||||
# List of member names, which should be excluded from the protected access
|
||||
# warning.
|
||||
exclude-protected=_asdict,
|
||||
_fields,
|
||||
_replace,
|
||||
_source,
|
||||
_make
|
||||
|
||||
# List of valid names for the first argument in a class method.
|
||||
valid-classmethod-first-arg=cls,
|
||||
class_
|
||||
|
||||
# List of valid names for the first argument in a metaclass class method.
|
||||
valid-metaclass-classmethod-first-arg=mcs
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
gunicorn==20.1.0; python_version > '3.0'
|
||||
gunicorn==19.10.0; python_version < '3.0'
|
||||
uvicorn~=0.17.0
|
||||
fastapi~=0.111.1
|
||||
fastapi-utilities==0.2.0
|
||||
cloudpickle==2.2.1
|
||||
grpcio==1.66.2
|
||||
urllib3==1.26.16
|
||||
typing-inspect==0.9.0
|
||||
typing_extensions==4.12.2
|
||||
pydantic==2.9.2
|
||||
requests==2.31.0
|
||||
|
||||
langchain==0.3.2
|
||||
langchain-community==0.3.1
|
||||
langchain-core==0.3.9
|
||||
langchain-google-vertexai==2.0.4
|
||||
|
||||
google-cloud-aiplatform==1.69.0
|
||||
google-cloud-bigquery==3.26.0
|
||||
google-cloud-tasks==2.16.5
|
||||
google-cloud-logging==3.11.2
|
||||
google-cloud-speech==2.27.0
|
||||
google-cloud-discoveryengine==0.13.4
|
||||
|
||||
pylint
|
||||
black
|
||||
pytest
|
||||
pytest-cov
|
||||
pytest-watch
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Utility functions for setting up Google BigQuery datasets and tables."""
|
||||
|
||||
from typing import List
|
||||
from google.cloud.bigquery import Client as BigQueryClient, Table, SchemaField, TableReference
|
||||
|
||||
bigquery_client = BigQueryClient()
|
||||
PROJECT_ID = bigquery_client.project
|
||||
|
||||
|
||||
def create_dataset(dataset_name: str):
|
||||
"""Deletes an existing BigQuery dataset and recreates it.
|
||||
|
||||
If the dataset already exists, it and all its contents
|
||||
will be deleted first.
|
||||
Then, a new empty dataset with the specified name is created.
|
||||
|
||||
Args:
|
||||
dataset_name: The name for the BigQuery dataset.
|
||||
"""
|
||||
dataset_id = f"{PROJECT_ID}.{dataset_name}"
|
||||
try:
|
||||
bigquery_client.delete_dataset(
|
||||
dataset_id, delete_contents=True, not_found_ok=True
|
||||
)
|
||||
print(f"Dataset {dataset_id} deleted (if it existed).")
|
||||
except Exception as e:
|
||||
print(f"Error deleting dataset {dataset_id}: {e}")
|
||||
|
||||
print(f"Creating dataset {dataset_id}...")
|
||||
bigquery_client.create_dataset(dataset_name, exists_ok=True) # exists_ok=True in case delete failed but it exists
|
||||
print(f"Dataset {dataset_id} ensured.")
|
||||
|
||||
|
||||
def create_table(dataset: str, table_name: str, schema: List[SchemaField]):
|
||||
"""Creates a BigQuery table within a specified dataset.
|
||||
If the table already exists, it will be deleted and recreated.
|
||||
|
||||
Args:
|
||||
dataset: The name of the dataset where the table will be created.
|
||||
table_name: The name for the new BigQuery table.
|
||||
schema: A list of SchemaField objects defining the table's structure.
|
||||
"""
|
||||
table_id_full = f"{PROJECT_ID}.{dataset}.{table_name}"
|
||||
table_ref = TableReference.from_string(table_id_full)
|
||||
|
||||
try:
|
||||
bigquery_client.delete_table(table_ref, not_found_ok=True)
|
||||
print(f"Table {table_id_full} deleted (if it existed).")
|
||||
except Exception as e:
|
||||
print(f"Notice: Could not delete table {table_id_full} (may not exist or other issue): {e}")
|
||||
|
||||
print(f"Creating table {table_id_full}...")
|
||||
bq_table = Table(table_ref, schema=schema)
|
||||
bigquery_client.create_table(bq_table)
|
||||
print(f"Table {table_id_full} created successfully.")
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from google.cloud import discoveryengine_v1 as discoveryengine
|
||||
from google.api_core.exceptions import AlreadyExists, NotFound
|
||||
|
||||
def create_vertex_ai_datastore(
|
||||
project_id: str,
|
||||
location: str,
|
||||
datastore_id: str,
|
||||
datastore_display_name: str,
|
||||
):
|
||||
"""Creates a Vertex AI Search Datastore if it doesn't exist."""
|
||||
client = discoveryengine.DataStoreServiceClient()
|
||||
parent = client.collection_path(project_id, location, "default_collection")
|
||||
datastore_name_full = client.data_store_path(project_id, location, datastore_id)
|
||||
|
||||
try:
|
||||
ds = client.get_data_store(name=datastore_name_full)
|
||||
print(f"Datastore '{datastore_id}' already exists in location '{location}': {ds.name}")
|
||||
return ds
|
||||
except NotFound:
|
||||
print(f"Datastore '{datastore_id}' not found in location '{location}'. Attempting to create...")
|
||||
|
||||
datastore = discoveryengine.DataStore(
|
||||
display_name=datastore_display_name,
|
||||
industry_vertical=discoveryengine.IndustryVertical.GENERIC,
|
||||
content_config=discoveryengine.DataStore.ContentConfig.CONTENT_REQUIRED,
|
||||
)
|
||||
|
||||
try:
|
||||
operation = client.create_data_store(
|
||||
parent=parent,
|
||||
data_store=datastore,
|
||||
data_store_id=datastore_id,
|
||||
)
|
||||
print(f"Waiting for Datastore '{datastore_id}' creation (LRO: {operation.operation.name})...")
|
||||
created_datastore = operation.result(timeout=300) # 5 minutes timeout
|
||||
print(f"Successfully created Datastore: {created_datastore.name}")
|
||||
return created_datastore
|
||||
except AlreadyExists:
|
||||
print(f"Datastore '{datastore_id}' creation reported AlreadyExists. Fetching existing.")
|
||||
return client.get_data_store(name=datastore_name_full)
|
||||
except Exception as e:
|
||||
print(f"Error creating Datastore '{datastore_id}': {e}")
|
||||
raise
|
||||
|
||||
def import_documents_to_datastore(
|
||||
project_id: str,
|
||||
location: str,
|
||||
datastore_id: str,
|
||||
gcs_uri: str,
|
||||
):
|
||||
"""Imports documents from GCS into the specified Datastore."""
|
||||
client = discoveryengine.DocumentServiceClient()
|
||||
parent_branch = client.branch_path(
|
||||
project=project_id,
|
||||
location=location,
|
||||
data_store=datastore_id,
|
||||
branch="default_branch",
|
||||
)
|
||||
|
||||
request = discoveryengine.ImportDocumentsRequest(
|
||||
parent=parent_branch,
|
||||
gcs_source=discoveryengine.GcsSource(input_uris=[gcs_uri], data_schema="content"),
|
||||
reconciliation_mode=discoveryengine.ImportDocumentsRequest.ReconciliationMode.INCREMENTAL
|
||||
)
|
||||
|
||||
try:
|
||||
print(f"Starting document import from '{gcs_uri}' into Datastore '{datastore_id}'...")
|
||||
operation = client.import_documents(request=request)
|
||||
print(f"Waiting for document import to complete (LRO: {operation.operation.name}). This may take several minutes...")
|
||||
response = operation.result(timeout=1800) # 30 minutes timeout
|
||||
|
||||
if response.error_samples and len(response.error_samples) > 0:
|
||||
print(f"Document import completed with errors. Error Config {response.error_config}")
|
||||
for i, error_sample in enumerate(response.error_samples):
|
||||
print(f" Error sample {i+1}: {error_sample.message}")
|
||||
raise Exception("Document import failed with errors", response)
|
||||
else:
|
||||
print(f"Successfully imported documents.")
|
||||
return response
|
||||
except Exception as e:
|
||||
print(f"Error during document import for Datastore '{datastore_id}': {e}")
|
||||
raise
|
||||
|
||||
def create_vertex_ai_engine(
|
||||
project_id: str,
|
||||
location: str,
|
||||
engine_id: str,
|
||||
engine_display_name: str,
|
||||
datastore_ids_list: list[str],
|
||||
):
|
||||
"""Creates a Vertex AI Search Engine (App) if it doesn't exist."""
|
||||
client = discoveryengine.EngineServiceClient()
|
||||
parent_collection = client.collection_path(project_id, location, "default_collection")
|
||||
engine_name_full = client.engine_path(project_id, location, "default_collection", engine_id)
|
||||
|
||||
try:
|
||||
eng = client.get_engine(name=engine_name_full)
|
||||
print(f"Engine '{engine_id}' already exists in location '{location}': {eng.name}")
|
||||
return eng
|
||||
except NotFound:
|
||||
print(f"Engine '{engine_id}' not found in location '{location}'. Attempting to create...")
|
||||
|
||||
engine_config = discoveryengine.Engine(
|
||||
display_name=engine_display_name,
|
||||
solution_type=discoveryengine.SolutionType.SOLUTION_TYPE_SEARCH,
|
||||
data_store_ids=datastore_ids_list,
|
||||
common_config=discoveryengine.Engine.CommonConfig(company_name="QuickBot App"),
|
||||
search_engine_config=discoveryengine.Engine.SearchEngineConfig(
|
||||
search_tier="SEARCH_TIER_STANDARD",
|
||||
search_add_ons=["SEARCH_ADD_ON_LLM"]
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
operation = client.create_engine(
|
||||
parent=parent_collection,
|
||||
engine=engine_config,
|
||||
engine_id=engine_id,
|
||||
)
|
||||
print(f"Waiting for Engine '{engine_id}' creation (LRO: {operation.operation.name})...")
|
||||
created_engine = operation.result(timeout=600) # 10 minutes timeout
|
||||
print(f"Successfully created Engine: {created_engine.name}")
|
||||
return created_engine
|
||||
except AlreadyExists:
|
||||
print(f"Engine '{engine_id}' creation reported AlreadyExists. Fetching existing.")
|
||||
return client.get_engine(name=engine_name_full)
|
||||
except Exception as e:
|
||||
print(f"Error creating Engine '{engine_id}': {e}")
|
||||
raise
|
||||
@@ -0,0 +1,120 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Initializes the BigQuery and Vertex AI Search environment for the application.
|
||||
|
||||
This script performs the following actions:
|
||||
1. Retrieves configuration from environment variables or uses defaults.
|
||||
2. Creates the specified BigQuery dataset if it doesn't already exist.
|
||||
3. Creates the 'search_applications' table within that dataset.
|
||||
4. Creates a Vertex AI Search Datastore if it doesn't already exist.
|
||||
5. Imports documents from a specified GCS bucket into the Datastore.
|
||||
6. Creates a Vertex AI Search Engine (App) linked to the Datastore.
|
||||
|
||||
Usage:
|
||||
Run this script directly (e.g., `python setup.py`).
|
||||
Set environment variables to override defaults:
|
||||
- 'BIG_QUERY_DATASET'
|
||||
- 'GOOGLE_CLOUD_PROJECT'
|
||||
- 'VERTEX_AI_SEARCH_LOCATION'
|
||||
- 'VERTEX_AI_DATASTORE_ID'
|
||||
- 'VERTEX_AI_ENGINE_ID'
|
||||
"""
|
||||
|
||||
from os import getenv
|
||||
from scripts.big_query_setup import create_dataset, create_table
|
||||
from src.service.search_application import SEARCH_APPLICATION_TABLE
|
||||
from src.model.search import SearchApplication
|
||||
from scripts.vertexai_search_setup import create_vertex_ai_datastore, create_vertex_ai_engine, import_documents_to_datastore
|
||||
|
||||
|
||||
def main():
|
||||
# 1. BigQuery Setup
|
||||
print("--- Setting up BigQuery ---")
|
||||
BIG_QUERY_DATASET = getenv("BIG_QUERY_DATASET", "quickbot_default_bq_dataset")
|
||||
GCLOUD_PROJECT = getenv("GCLOUD_PROJECT", "my-gcloud-project")
|
||||
create_dataset(BIG_QUERY_DATASET)
|
||||
create_table(
|
||||
BIG_QUERY_DATASET, SEARCH_APPLICATION_TABLE, SearchApplication.__schema__()
|
||||
)
|
||||
|
||||
# 2. Vertex AI Search Setup
|
||||
print("--- Setting up Vertex AI Search ---")
|
||||
VERTEX_AI_LOCATION = getenv("VERTEX_AI_LOCATION", "global")
|
||||
VERTEX_AI_DATASTORE_ID = getenv("VERTEX_AI_DATASTORE_ID", "quickbot_alphabet_pdfs_ds")
|
||||
VERTEX_AI_ENGINE_ID = getenv("VERTEX_AI_ENGINE_ID", "quickbot_alphabet_search_engine")
|
||||
GCS_SOURCE_URI = "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/*.pdf"
|
||||
DATASTORE_DISPLAY_NAME_PREFIX = "Alphabet Investor Docs DS"
|
||||
ENGINE_DISPLAY_NAME_PREFIX = "Alphabet Investor Engine"
|
||||
|
||||
datastore_display_name = f"{DATASTORE_DISPLAY_NAME_PREFIX} ({VERTEX_AI_DATASTORE_ID})"
|
||||
engine_display_name = f"{ENGINE_DISPLAY_NAME_PREFIX} ({VERTEX_AI_ENGINE_ID})"
|
||||
try:
|
||||
# Create/Get Datastore
|
||||
print(f"Attempting to create/get Datastore '{VERTEX_AI_DATASTORE_ID}' in project '{GCLOUD_PROJECT}' location '{VERTEX_AI_LOCATION}'...")
|
||||
datastore = create_vertex_ai_datastore(
|
||||
GCLOUD_PROJECT, VERTEX_AI_LOCATION, VERTEX_AI_DATASTORE_ID, datastore_display_name
|
||||
)
|
||||
if not datastore:
|
||||
print("Datastore creation/retrieval failed. Aborting further Vertex AI Search setup.")
|
||||
print("--- Application setup finished (with errors) ---")
|
||||
raise
|
||||
print(f"Successfully ensured Datastore exists: {datastore.name}")
|
||||
|
||||
# Import documents into Datastore
|
||||
# Note: This will attempt to import documents every time the script runs.
|
||||
# For production, you might want to add a check to skip this if documents
|
||||
# are already present or if a previous import was successful.
|
||||
print(f"\nAttempting to import documents from '{GCS_SOURCE_URI}' into datastore: {datastore.name}")
|
||||
import_documents_to_datastore(
|
||||
GCLOUD_PROJECT, VERTEX_AI_LOCATION, VERTEX_AI_DATASTORE_ID, GCS_SOURCE_URI
|
||||
)
|
||||
# Note: Document import can take a long time. The script waits.
|
||||
print("Document import process initiated/completed.\n")
|
||||
|
||||
# Create/Get Engine
|
||||
print(f"Attempting to create/get Engine '{VERTEX_AI_ENGINE_ID}' in project '{GCLOUD_PROJECT}' location '{VERTEX_AI_LOCATION}'...")
|
||||
# The create_vertex_ai_engine function expects a list of datastore IDs (not full resource names).
|
||||
engine = create_vertex_ai_engine(
|
||||
GCLOUD_PROJECT,
|
||||
VERTEX_AI_LOCATION,
|
||||
VERTEX_AI_ENGINE_ID,
|
||||
engine_display_name,
|
||||
[VERTEX_AI_DATASTORE_ID] # Pass the Datastore ID string
|
||||
)
|
||||
|
||||
if not engine:
|
||||
print("Engine creation/retrieval failed.")
|
||||
print("--- Application setup finished (with errors) ---")
|
||||
raise
|
||||
print(f"Successfully ensured Engine exists: {engine.name}")
|
||||
|
||||
print("\nVertex AI Search setup completed successfully.")
|
||||
|
||||
except Exception as e:
|
||||
print(f"A critical error occurred during the setup process: {e}")
|
||||
import traceback
|
||||
print("Detailed traceback:")
|
||||
print(traceback.format_exc())
|
||||
# If running in Docker build, exiting with non-zero will fail the build
|
||||
import sys
|
||||
sys.exit(1)
|
||||
|
||||
print("\n--- Application setup finished ---")
|
||||
print("\nSuccess! All resources should now be configured.\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""API endpoints for managing and performing document searches."""
|
||||
|
||||
from fastapi import APIRouter, Request, HTTPException, Response
|
||||
from google.cloud import storage
|
||||
from pydantic import BaseModel
|
||||
from src.model.http_status import BadRequest
|
||||
from src.model.search import CreateSearchRequest, SearchApplication
|
||||
from src.service.engine import EngineService
|
||||
from src.service.search import SearchService
|
||||
from src.service.search_application import SearchApplicationService
|
||||
|
||||
|
||||
class SignedUrlRequest(BaseModel):
|
||||
"""Request model for fetching a document directly from GCS."""
|
||||
|
||||
gcs_url: str
|
||||
|
||||
|
||||
storage_client = storage.Client()
|
||||
|
||||
router = APIRouter(
|
||||
prefix="/api/search",
|
||||
tags=["searches"],
|
||||
responses={404: {"description": "Not found"}},
|
||||
)
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def search(item: CreateSearchRequest):
|
||||
"""
|
||||
Performs a search using the configured Search Application.
|
||||
|
||||
Args:
|
||||
item: The search request containing the search term.
|
||||
|
||||
Raises:
|
||||
BadRequest: If no Search Application is configured for the project.
|
||||
|
||||
Returns:
|
||||
The search results from the SearchService.
|
||||
"""
|
||||
service = SearchApplicationService()
|
||||
search_application = service.get()
|
||||
if not search_application:
|
||||
raise BadRequest(detail="No Search Application found on project")
|
||||
|
||||
service = SearchService(
|
||||
search_application,
|
||||
)
|
||||
return service.search(item.term)
|
||||
|
||||
|
||||
@router.get("/engines")
|
||||
async def get_all_engines():
|
||||
"""Retrieves all available Search Engines."""
|
||||
service = EngineService()
|
||||
return service.get_all()
|
||||
|
||||
|
||||
@router.get("/application")
|
||||
async def get_search_application():
|
||||
"""Retrieves the currently configured Search Application."""
|
||||
service = SearchApplicationService()
|
||||
return service.get()
|
||||
|
||||
|
||||
@router.post("/application")
|
||||
async def create_search_application(search_application: SearchApplication):
|
||||
"""
|
||||
Creates a new Search Application configuration.
|
||||
|
||||
Args:
|
||||
search_application: The details of the Search Application to create.
|
||||
|
||||
Returns:
|
||||
The created Search Application configuration.
|
||||
"""
|
||||
service = SearchApplicationService()
|
||||
return service.create(search_application)
|
||||
|
||||
|
||||
@router.put("/application/{engine_id}")
|
||||
async def update_search_application(
|
||||
engine_id: str, search_application: SearchApplication
|
||||
):
|
||||
"""
|
||||
Updates an existing Search Application configuration.
|
||||
|
||||
Args:
|
||||
engine_id: The ID of the engine associated with
|
||||
the application to update.
|
||||
search_application: The updated details for the Search Application.
|
||||
|
||||
Returns:
|
||||
The updated Search Application configuration.
|
||||
"""
|
||||
service = SearchApplicationService()
|
||||
return service.update(engine_id, search_application)
|
||||
|
||||
|
||||
@router.post("/doc")
|
||||
async def get_document(request: Request, response_model=None):
|
||||
"""
|
||||
Fetches a document directly from GCS and returns its content.
|
||||
|
||||
Expects a JSON body with a 'gcs_url' field specifying the full GCS path
|
||||
(e.g., "gs://your-bucket-name/your-file.pdf").
|
||||
|
||||
Args:
|
||||
request: The incoming FastAPI request object.
|
||||
|
||||
Raises:
|
||||
HTTPException: If the GCS URL is invalid or the file cannot be fetched.
|
||||
|
||||
Returns:
|
||||
A FastAPI Response object containing the raw PDF content.
|
||||
"""
|
||||
try:
|
||||
req_body = await request.json()
|
||||
signed_url_request = SignedUrlRequest(**req_body)
|
||||
gcs_url = signed_url_request.gcs_url
|
||||
|
||||
bucket_name = gcs_url.split("/")[2]
|
||||
object_name = "/".join(gcs_url.split("/")[3:])
|
||||
|
||||
# Get a reference to the blob (PDF file)
|
||||
bucket = storage_client.bucket(bucket_name)
|
||||
blob = bucket.blob(object_name)
|
||||
|
||||
# Download the blob content as bytes
|
||||
pdf_content = blob.download_as_bytes()
|
||||
|
||||
# Return the PDF content with appropriate headers
|
||||
return Response(content=pdf_content, media_type="application/pdf")
|
||||
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
status_code=500, detail=f"Error fetching PDF: {str(e)}"
|
||||
)
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Defines custom HTTP exception classes for the FastAPI application.
|
||||
|
||||
This module provides reusable exception classes that inherit from FastAPI's
|
||||
HTTPException, allowing for standardized error responses with specific
|
||||
status codes and detail messages.
|
||||
"""
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
||||
class ResourceAlreadyExists(HTTPException):
|
||||
"""
|
||||
Custom exception for HTTP 400 Bad Request errors.
|
||||
|
||||
Used specifically when an attempt is made to create a resource that
|
||||
already exists (e.g., creating a duplicate configuration).
|
||||
Defaults to status code 400.
|
||||
"""
|
||||
|
||||
def __init__(self, detail="Resource already exists"):
|
||||
"""Initializes the exception with a default detail message."""
|
||||
super().__init__(status_code=400, detail=detail)
|
||||
|
||||
|
||||
class BadRequest(HTTPException):
|
||||
"""
|
||||
Custom exception for general HTTP 400 Bad Request errors.
|
||||
|
||||
Can be used for various client-side errors like invalid input,
|
||||
missing required data, or violating business logic rules before
|
||||
processing. Defaults to status code 400.
|
||||
|
||||
Note: The default detail message "Resource already exists" seems
|
||||
inconsistent with a general BadRequest. Consider changing it to
|
||||
a more generic message like "Bad Request" or requiring a specific
|
||||
detail message upon instantiation.
|
||||
"""
|
||||
def __init__(self, detail="Resource already exists"):
|
||||
super().__init__(status_code=400, detail=detail)
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Defines data models for search operations and configurations.
|
||||
|
||||
This module includes Pydantic models for API requests/responses related to
|
||||
search applications and engines, as well as dataclasses for representing
|
||||
search results internally. It also handles fetching the default Google Cloud
|
||||
Project ID.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional
|
||||
from pydantic import BaseModel
|
||||
from google.cloud.bigquery import SchemaField, Row
|
||||
from google.api_core.client_options import ClientOptions
|
||||
import google.auth
|
||||
|
||||
_, PROJECT_ID = google.auth.default()
|
||||
|
||||
|
||||
class CreateSearchRequest(BaseModel):
|
||||
"""Request model for initiating a search."""
|
||||
term: str
|
||||
|
||||
|
||||
class SearchApplication(BaseModel):
|
||||
"""Represents the configuration for a Discovery Engine
|
||||
Search Application."""
|
||||
engine_id: str
|
||||
region: str
|
||||
|
||||
@classmethod
|
||||
def __schema__(cls) -> List[SchemaField]:
|
||||
"""Defines the BigQuery schema for storing SearchApplication data."""
|
||||
return [
|
||||
SchemaField("engine_id", "STRING", mode="REQUIRED"),
|
||||
SchemaField("region", "STRING", mode="REQUIRED"),
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def from_row(cls, row: Row) -> 'SearchApplication':
|
||||
"""Creates a SearchApplication instance from a BigQuery Row object.
|
||||
|
||||
Args:
|
||||
row: The BigQuery Row object. Assumes row contains fields matching the schema.
|
||||
|
||||
Returns:
|
||||
A SearchApplication instance.
|
||||
"""
|
||||
# Access by field name for robustness, assuming schema matches
|
||||
return cls(engine_id=row["engine_id"], region=row["region"])
|
||||
|
||||
def to_dict(self):
|
||||
"""Converts the SearchApplication instance to a dictionary."""
|
||||
return {
|
||||
"engine_id": self.engine_id,
|
||||
"region": self.region,
|
||||
}
|
||||
|
||||
def get_client_options(self) -> Optional[ClientOptions]:
|
||||
"""Generates API client options based on the application's region.
|
||||
|
||||
Returns:
|
||||
ClientOptions configured with the regional endpoint, or None if
|
||||
the region is 'global'.
|
||||
"""
|
||||
return (
|
||||
ClientOptions(
|
||||
api_endpoint=f"{self.region}-discoveryengine.googleapis.com"
|
||||
)
|
||||
if self.region != "global"
|
||||
else None
|
||||
)
|
||||
|
||||
def get_serving_config(self) -> str:
|
||||
"""Constructs the full serving config path for the Discovery Engine API.
|
||||
|
||||
Returns:
|
||||
The formatted serving config string.
|
||||
"""
|
||||
serving_config = f"projects/{PROJECT_ID}/locations/{self.region}"
|
||||
serving_config += f"/collections/default_collection/engines/{self.engine_id}"
|
||||
serving_config += "/servingConfigs/default_config"
|
||||
return serving_config
|
||||
|
||||
class Engine(BaseModel):
|
||||
"""Represents a discovered Discovery Engine instance."""
|
||||
name: str
|
||||
engine_id: str
|
||||
region: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResult:
|
||||
"""Represents a single document result from a search query."""
|
||||
document_id: str
|
||||
title: str
|
||||
snippet: str
|
||||
link: Optional[str] = None
|
||||
content: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SearchResultsWithSummary:
|
||||
"""Represents the complete results of a search, including a summary."""
|
||||
results: List[SearchResult]
|
||||
summary: Optional[str] = None
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
+318
@@ -0,0 +1,318 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Provides a repository class for interacting with Google BigQuery.
|
||||
|
||||
This module defines the `BigQueryRepository` class, which encapsulates
|
||||
common BigQuery operations like running queries, fetching rows by ID,
|
||||
inserting, updating, and deleting rows. It relies on the 'BIG_QUERY_DATASET'
|
||||
environment variable to target the correct dataset.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
from decimal import Decimal
|
||||
from os import getenv
|
||||
from typing import Dict, List, Any, Optional, Tuple
|
||||
from google.cloud.bigquery import (
|
||||
Client,
|
||||
QueryJobConfig,
|
||||
ScalarQueryParameter,
|
||||
ArrayQueryParameter,
|
||||
TableReference,
|
||||
DatasetReference
|
||||
)
|
||||
from google.cloud.bigquery.table import RowIterator
|
||||
from google.cloud.exceptions import GoogleCloudError
|
||||
|
||||
_BIG_QUERY_DATASET_ENV = getenv("BIG_QUERY_DATASET")
|
||||
if not _BIG_QUERY_DATASET_ENV:
|
||||
raise ValueError(
|
||||
"The BIG_QUERY_DATASET environment variable is not set. "
|
||||
"This is required for the BigQueryRepository to function."
|
||||
)
|
||||
BIG_QUERY_DATASET: str = _BIG_QUERY_DATASET_ENV
|
||||
|
||||
|
||||
class BigQueryRepository:
|
||||
"""
|
||||
A repository class for simplifying interactions with Google BigQuery,
|
||||
using parameterized queries to prevent SQL injection.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initializes the BigQuery client and project ID."""
|
||||
self.client: Client = Client()
|
||||
self.project_id: str = self.client.project
|
||||
self.dataset_id: str = BIG_QUERY_DATASET
|
||||
|
||||
def _get_table_ref(self, table_id: str) -> TableReference:
|
||||
"""Helper to get a TableReference object."""
|
||||
dataset_ref = DatasetReference(self.project_id, self.dataset_id)
|
||||
return TableReference(dataset_ref, table_id)
|
||||
|
||||
def _validate_column_name(self, column_name: str):
|
||||
"""Basic validation for column names."""
|
||||
if not column_name.replace("_", "").isalnum():
|
||||
raise ValueError(f"Invalid column name: {column_name}")
|
||||
|
||||
def _create_scalar_query_parameter(self, name: str, value: Any) -> ScalarQueryParameter:
|
||||
"""
|
||||
Creates a ScalarQueryParameter by inferring the BigQuery type from the value.
|
||||
"""
|
||||
if isinstance(value, bool):
|
||||
return ScalarQueryParameter(name, "BOOL", value)
|
||||
if isinstance(value, int):
|
||||
return ScalarQueryParameter(name, "INT64", value)
|
||||
if isinstance(value, float):
|
||||
return ScalarQueryParameter(name, "FLOAT64", value)
|
||||
if isinstance(value, str):
|
||||
return ScalarQueryParameter(name, "STRING", value)
|
||||
if isinstance(value, datetime.datetime):
|
||||
return ScalarQueryParameter(name, "TIMESTAMP", value)
|
||||
if isinstance(value, datetime.date):
|
||||
return ScalarQueryParameter(name, "DATE", value)
|
||||
if isinstance(value, Decimal):
|
||||
return ScalarQueryParameter(name, "NUMERIC", value)
|
||||
if value is None:
|
||||
# For None, the type of the column in BQ will ultimately determine how NULL is handled.
|
||||
# Using STRING type for the parameter is a common safe default.
|
||||
return ScalarQueryParameter(name, "STRING", None) # Default to STRING for None
|
||||
|
||||
# Fallback: attempt to convert other types to string.
|
||||
try:
|
||||
str_value = str(value)
|
||||
return ScalarQueryParameter(name, "STRING", str_value)
|
||||
except Exception as e:
|
||||
raise TypeError(
|
||||
f"Value for param '{name}' of type {type(value)} "
|
||||
f"could not be converted to a supported BigQuery scalar type: {e}"
|
||||
)
|
||||
|
||||
def run_query(self, query: str, job_config: Optional[QueryJobConfig] = None) -> RowIterator:
|
||||
"""
|
||||
Executes a BigQuery SQL query and returns the results.
|
||||
|
||||
Args:
|
||||
query: The SQL query string to execute.
|
||||
job_config: Optional QueryJobConfig for parameterized queries.
|
||||
|
||||
Returns:
|
||||
A RowIterator object to iterate over the query results.
|
||||
|
||||
Raises:
|
||||
google.cloud.exceptions.GoogleCloudError: If the query fails.
|
||||
"""
|
||||
try:
|
||||
query_job = self.client.query(query, job_config=job_config)
|
||||
return query_job.result()
|
||||
except GoogleCloudError as e:
|
||||
print(f"BigQuery Error: {e}")
|
||||
print(f"Query: {query}")
|
||||
if job_config and job_config.query_parameters:
|
||||
print(f"Parameters: {job_config.query_parameters}")
|
||||
raise
|
||||
|
||||
def get_row_by_id(self, table_id: str, id_column: str, id_value: Any, id_value_type: str = "STRING") -> RowIterator:
|
||||
"""
|
||||
Retrieves a single row from a table based on its ID using parameterized query.
|
||||
|
||||
Args:
|
||||
table_id: The ID of the table (without dataset prefix).
|
||||
id_column: The name of the column containing the ID.
|
||||
id_value: The specific ID value to search for.
|
||||
id_value_type: The BigQuery type of the ID value (e.g., "STRING", "INT64").
|
||||
|
||||
Returns:
|
||||
A RowIterator containing the matching row(s).
|
||||
"""
|
||||
self._validate_column_name(id_column)
|
||||
query = f"""
|
||||
SELECT * FROM `{self.project_id}.{self.dataset_id}.{table_id}`
|
||||
WHERE `{id_column}` = @id_value;
|
||||
"""
|
||||
job_config = QueryJobConfig(
|
||||
query_parameters=[
|
||||
ScalarQueryParameter("id_value", id_value_type, id_value)
|
||||
]
|
||||
)
|
||||
return self.run_query(query, job_config=job_config)
|
||||
|
||||
def insert_rows_json(self, table_id: str, rows: List[Dict[str, Any]]) -> List[Dict]:
|
||||
"""
|
||||
Inserts multiple rows into the specified table using JSON.
|
||||
This is the recommended method for insertions due to its safety and efficiency.
|
||||
|
||||
Args:
|
||||
table_id: The ID of the table.
|
||||
rows: A list of dictionaries, where each dictionary represents a row
|
||||
(column names as keys, values as row data).
|
||||
|
||||
Returns:
|
||||
A list of error dictionaries if any rows failed to insert,
|
||||
otherwise an empty list.
|
||||
"""
|
||||
table_ref = self._get_table_ref(table_id)
|
||||
errors_from_client = self.client.insert_rows_json(table_ref, rows)
|
||||
errors_list: List[Dict[str, Any]] = list(errors_from_client) # type: ignore
|
||||
if errors_list:
|
||||
print(f"Errors inserting rows into {table_id}: {errors_list}")
|
||||
return errors_list
|
||||
|
||||
def insert_row(self, table_id: str, column_names: List[str], values: Tuple[Any, ...]) -> None:
|
||||
"""
|
||||
Inserts a new row into the specified table using a parameterized query.
|
||||
Prefer `insert_rows_json` for multiple rows or complex data.
|
||||
|
||||
Args:
|
||||
table_id: The ID of the table.
|
||||
column_names: A list of column names for the insert.
|
||||
values: A tuple of values corresponding to the column_names.
|
||||
|
||||
Raises:
|
||||
ValueError: If column names are invalid or counts don't match.
|
||||
TypeError: If a value in `values` cannot be mapped to a BigQuery scalar type.
|
||||
"""
|
||||
if not column_names or not values:
|
||||
raise ValueError("Column names and values cannot be empty.")
|
||||
if len(column_names) != len(values):
|
||||
raise ValueError("Number of column names must match number of values.")
|
||||
|
||||
for col_name in column_names:
|
||||
self._validate_column_name(col_name)
|
||||
|
||||
cols_str = ", ".join(f"`{col}`" for col in column_names)
|
||||
placeholders = ", ".join([f"@param{i}" for i in range(len(values))])
|
||||
|
||||
query_params: List[ScalarQueryParameter] = []
|
||||
for i, current_value in enumerate(values):
|
||||
param_name = f"param{i}"
|
||||
query_params.append(self._create_scalar_query_parameter(param_name, current_value))
|
||||
|
||||
query = f"""
|
||||
INSERT INTO `{self.project_id}.{self.dataset_id}.{table_id}` ({cols_str})
|
||||
VALUES ({placeholders});
|
||||
"""
|
||||
job_config = QueryJobConfig(query_parameters=query_params)
|
||||
self.run_query(query, job_config=job_config)
|
||||
|
||||
def delete_multiple_rows_by_id(
|
||||
self, table_id: str, id_column: str, ids: List[Any], id_value_type: str = "STRING"
|
||||
) -> None:
|
||||
"""
|
||||
Deletes multiple rows from a table based on a list of IDs using parameterized query.
|
||||
|
||||
Args:
|
||||
table_id: The ID of the table.
|
||||
id_column: The name of the column containing the IDs.
|
||||
ids: A list of ID values to delete.
|
||||
id_value_type: The BigQuery type of the ID values.
|
||||
"""
|
||||
if not ids:
|
||||
return # Nothing to delete
|
||||
self._validate_column_name(id_column)
|
||||
|
||||
query = f"""
|
||||
DELETE FROM `{self.project_id}.{self.dataset_id}.{table_id}`
|
||||
WHERE `{id_column}` IN UNNEST(@ids);
|
||||
"""
|
||||
job_config = QueryJobConfig(
|
||||
query_parameters=[
|
||||
ArrayQueryParameter("ids", id_value_type, ids)
|
||||
]
|
||||
)
|
||||
self.run_query(query, job_config=job_config)
|
||||
|
||||
def update_row_by_id(
|
||||
self,
|
||||
table_id: str,
|
||||
id_column: str,
|
||||
id_value: Any,
|
||||
column_values: Dict[str, Any],
|
||||
id_value_type: str = "STRING",
|
||||
) -> None:
|
||||
"""
|
||||
Updates specific columns of a row identified by its ID using parameterized query.
|
||||
|
||||
Args:
|
||||
table_id: The ID of the table.
|
||||
id_column: The name of the column containing the ID.
|
||||
id_value: The specific ID value of the row to update.
|
||||
column_values: A dictionary where keys are column names and
|
||||
values are the new values.
|
||||
id_value_type: The BigQuery type of the ID value.
|
||||
"""
|
||||
if not column_values:
|
||||
return
|
||||
self._validate_column_name(id_column)
|
||||
|
||||
set_clauses = []
|
||||
query_params_list: List[ScalarQueryParameter] = [
|
||||
ScalarQueryParameter("id_value", id_value_type, id_value)
|
||||
]
|
||||
|
||||
param_idx = 0
|
||||
for col, current_value in column_values.items():
|
||||
self._validate_column_name(col)
|
||||
update_param_name = f"update_val_{param_idx}"
|
||||
set_clauses.append(f"`{col}` = @{update_param_name}")
|
||||
query_params_list.append(self._create_scalar_query_parameter(update_param_name, current_value))
|
||||
param_idx += 1
|
||||
|
||||
if not set_clauses: # Should not happen if column_values is not empty
|
||||
return
|
||||
|
||||
sets_str = ", ".join(set_clauses)
|
||||
query = f"""
|
||||
UPDATE `{self.project_id}.{self.dataset_id}.{table_id}`
|
||||
SET {sets_str}
|
||||
WHERE `{id_column}` = @id_value;
|
||||
"""
|
||||
job_config = QueryJobConfig(query_parameters=query_params_list)
|
||||
self.run_query(query, job_config=job_config)
|
||||
|
||||
def get_all_rows(self, table_id: str) -> RowIterator:
|
||||
"""
|
||||
Retrieves all rows from the specified table.
|
||||
|
||||
Args:
|
||||
table_id: The ID of the table.
|
||||
|
||||
Returns:
|
||||
A RowIterator containing all rows in the table.
|
||||
"""
|
||||
query = f"SELECT * FROM `{self.project_id}.{self.dataset_id}.{table_id}`"
|
||||
return self.run_query(query)
|
||||
|
||||
def delete_row_by_id(self, table_id: str, id_column: str, id_value: Any, id_value_type: str = "STRING") -> None:
|
||||
"""
|
||||
Deletes a single row from a table based on its ID using parameterized query.
|
||||
|
||||
Args:
|
||||
table_id: The ID of the table.
|
||||
id_column: The name of the column containing the ID.
|
||||
id_value: The specific ID value of the row to delete.
|
||||
id_value_type: The BigQuery type of the ID value.
|
||||
"""
|
||||
self._validate_column_name(id_column)
|
||||
query = f"""
|
||||
DELETE FROM `{self.project_id}.{self.dataset_id}.{table_id}`
|
||||
WHERE `{id_column}` = @id_value;
|
||||
"""
|
||||
job_config = QueryJobConfig(
|
||||
query_parameters=[
|
||||
ScalarQueryParameter("id_value", id_value_type, id_value)
|
||||
]
|
||||
)
|
||||
self.run_query(query, job_config=job_config)
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Service for interacting with Google Cloud Discovery Engine Engines."""
|
||||
|
||||
from typing import List
|
||||
from google.cloud.discoveryengine_v1 import (
|
||||
EngineServiceClient,
|
||||
ListEnginesRequest,
|
||||
)
|
||||
from google.api_core.client_options import ClientOptions
|
||||
|
||||
from src.model.search import PROJECT_ID, Engine
|
||||
|
||||
LOCATIONS = [
|
||||
"global",
|
||||
"us",
|
||||
]
|
||||
|
||||
|
||||
class EngineService:
|
||||
"""Provides methods to list Discovery Engine Engines."""
|
||||
|
||||
def get_all(self) -> List[Engine]:
|
||||
"""
|
||||
Retrieves all available Discovery Engines for the configured project
|
||||
across specified locations.
|
||||
|
||||
Fetches engines from predefined locations ('global', 'us') within the
|
||||
default collection.
|
||||
|
||||
Returns:
|
||||
A list of Engine objects, each containing details about a
|
||||
discovered engine. Returns an empty list if no engines are found
|
||||
or if an error occurs during API calls.
|
||||
|
||||
Raises:
|
||||
Prints error logs if API calls fail for a specific location.
|
||||
"""
|
||||
engines = []
|
||||
for location in LOCATIONS:
|
||||
client_options = (
|
||||
ClientOptions(
|
||||
api_endpoint=f"{location}-discoveryengine.googleapis.com"
|
||||
)
|
||||
if location != "global"
|
||||
else None
|
||||
)
|
||||
|
||||
# Create a client
|
||||
client = EngineServiceClient(client_options=client_options)
|
||||
list_engines = client.list_engines(
|
||||
ListEnginesRequest(
|
||||
parent=f"projects/{PROJECT_ID}/locations/{location}"
|
||||
"/collections/default_collection"
|
||||
)
|
||||
)
|
||||
for engine in list_engines.engines:
|
||||
engines.append(
|
||||
Engine(
|
||||
name=engine.display_name,
|
||||
engine_id=engine.name.split("/")[-1],
|
||||
region=location,
|
||||
)
|
||||
)
|
||||
return engines
|
||||
+144
@@ -0,0 +1,144 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Service for performing searches using Google Cloud Discovery Engine."""
|
||||
|
||||
from google.cloud.discoveryengine_v1 import SearchRequest, SearchServiceClient
|
||||
from src.model.search import (
|
||||
SearchApplication,
|
||||
SearchResult,
|
||||
SearchResultsWithSummary,
|
||||
)
|
||||
|
||||
CONTENT_SEARCH_SPEC = SearchRequest.ContentSearchSpec(
|
||||
snippet_spec=SearchRequest.ContentSearchSpec.SnippetSpec(
|
||||
return_snippet=True
|
||||
),
|
||||
extractive_content_spec=SearchRequest.ContentSearchSpec.ExtractiveContentSpec(
|
||||
max_extractive_answer_count=1
|
||||
),
|
||||
summary_spec=SearchRequest.ContentSearchSpec.SummarySpec(
|
||||
summary_result_count=5,
|
||||
include_citations=True,
|
||||
ignore_adversarial_query=True,
|
||||
ignore_non_summary_seeking_query=True,
|
||||
model_spec=SearchRequest.ContentSearchSpec.SummarySpec.ModelSpec(
|
||||
version="stable",
|
||||
),
|
||||
),
|
||||
)
|
||||
QUERY_EXPANSION_SPEC = SearchRequest.QueryExpansionSpec(
|
||||
condition=SearchRequest.QueryExpansionSpec.Condition.AUTO,
|
||||
)
|
||||
SPELL_CORRECTION_SPEC = SearchRequest.SpellCorrectionSpec(
|
||||
mode=SearchRequest.SpellCorrectionSpec.Mode.AUTO
|
||||
)
|
||||
|
||||
|
||||
class SearchService:
|
||||
"""
|
||||
Handles search operations using a configured
|
||||
Discovery Engine Search Application.
|
||||
|
||||
Initializes with a specific SearchApplication configuration and provides
|
||||
methods to perform searches against the corresponding engine.
|
||||
"""
|
||||
|
||||
def __init__(self, search_application: SearchApplication):
|
||||
"""
|
||||
Initializes the SearchService.
|
||||
|
||||
Args:
|
||||
search_application: The configuration object defining the target
|
||||
search engine, location, and serving config.
|
||||
"""
|
||||
self.search_client = SearchServiceClient(
|
||||
client_options=search_application.get_client_options()
|
||||
)
|
||||
self.serving_config = search_application.get_serving_config()
|
||||
|
||||
def search(self, term: str) -> SearchResultsWithSummary:
|
||||
"""
|
||||
Performs a search against the configured Discovery Engine.
|
||||
|
||||
Constructs a search request with predefined content specifications,
|
||||
query expansion, and spell correction settings. Parses the response
|
||||
to extract results, snippets, links, and a summary.
|
||||
|
||||
Args:
|
||||
term: The search query string.
|
||||
|
||||
Returns:
|
||||
A SearchResultsWithSummary object containing the search summary
|
||||
and a list of SearchResult objects. Returns an empty list of
|
||||
results and a default summary message if the search fails or
|
||||
yields no results.
|
||||
|
||||
Raises:
|
||||
Logs errors if the API call fails.
|
||||
"""
|
||||
request = SearchRequest(
|
||||
serving_config=self.serving_config,
|
||||
query=term,
|
||||
page_size=10,
|
||||
content_search_spec=CONTENT_SEARCH_SPEC,
|
||||
query_expansion_spec=QUERY_EXPANSION_SPEC,
|
||||
spell_correction_spec=SPELL_CORRECTION_SPEC,
|
||||
)
|
||||
|
||||
data = self.search_client.search(request)
|
||||
results = []
|
||||
|
||||
summary_text = (
|
||||
data.summary.summary_text
|
||||
if data.summary and data.summary.summary_text
|
||||
else "No summary available"
|
||||
)
|
||||
|
||||
# Process results
|
||||
for r in data.results:
|
||||
document = r.document
|
||||
derived_data = document.derived_struct_data
|
||||
gcs_link = derived_data.get("link").replace(
|
||||
"gs://", "https://storage.cloud.google.com/"
|
||||
)
|
||||
|
||||
# Extract snippet safely
|
||||
snippets = derived_data.get("snippets", [])
|
||||
snippet_text = (
|
||||
snippets[0].get("snippet", "No snippet available")
|
||||
if snippets
|
||||
else "No snippet available"
|
||||
)
|
||||
extractive_answers = derived_data.get("extractive_answers", [])
|
||||
content_text = (
|
||||
extractive_answers[0].get("content", "No content available")
|
||||
if extractive_answers
|
||||
else "No content available"
|
||||
)
|
||||
# Map to SearchResult
|
||||
mapped_result = SearchResult(
|
||||
document_id=document.id,
|
||||
title=derived_data.get("title", "Untitled"),
|
||||
snippet=snippet_text,
|
||||
link=gcs_link,
|
||||
content=content_text,
|
||||
)
|
||||
results.append(mapped_result)
|
||||
|
||||
response_result = SearchResultsWithSummary(
|
||||
summary=summary_text, results=results
|
||||
)
|
||||
|
||||
return response_result
|
||||
+119
@@ -0,0 +1,119 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Service for managing the Search Application configuration
|
||||
stored in BigQuery."""
|
||||
|
||||
from src.model.http_status import BadRequest
|
||||
from src.model.search import SearchApplication
|
||||
from src.repository.big_query import BigQueryRepository
|
||||
|
||||
SEARCH_APPLICATION_TABLE = "search_applications"
|
||||
SEARCH_APPLICATION_TABLE_ID_COLUMN = "engine_id"
|
||||
|
||||
|
||||
class SearchApplicationService:
|
||||
"""Handles business logic for Search Application configuration."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initializes the service with a BigQuery repository."""
|
||||
self.repository = BigQueryRepository()
|
||||
|
||||
def get(self):
|
||||
"""
|
||||
Retrieves the currently configured Search Application.
|
||||
|
||||
Assumes that there is at most one Search Application configuration
|
||||
stored in the BigQuery table. If multiple rows exist, it returns
|
||||
the last one processed.
|
||||
|
||||
Returns:
|
||||
A SearchApplication object if found, otherwise None.
|
||||
"""
|
||||
search_application = None
|
||||
results = self.repository.get_all_rows(SEARCH_APPLICATION_TABLE)
|
||||
# Convert RowIterator to list to check if it's empty or get the last item
|
||||
rows = list(results)
|
||||
if rows:
|
||||
# Assuming we take the last row if multiple exist, or the only one
|
||||
search_application = SearchApplication.from_row(rows[-1])
|
||||
|
||||
return search_application
|
||||
|
||||
def create(
|
||||
self, search_application: SearchApplication
|
||||
) -> SearchApplication:
|
||||
"""
|
||||
Creates a new Search Application configuration in BigQuery.
|
||||
|
||||
Ensures that no configuration already exists before creating a new one.
|
||||
|
||||
Args:
|
||||
search_application: The SearchApplication object to create.
|
||||
|
||||
Returns:
|
||||
The created SearchApplication object.
|
||||
|
||||
Raises:
|
||||
BadRequest: If a Search Application configuration already exists
|
||||
for the project.
|
||||
"""
|
||||
if self.get():
|
||||
raise BadRequest(
|
||||
detail="Search Application for this project already exists"
|
||||
)
|
||||
schema_fields = SearchApplication.__schema__()
|
||||
column_names = [field.name for field in schema_fields]
|
||||
|
||||
# Prepare values as a tuple, matching the order of schema_fields
|
||||
# Ensure the order here matches the order in SearchApplication.__schema__
|
||||
values_tuple = (search_application.engine_id, search_application.region)
|
||||
|
||||
self.repository.insert_row(
|
||||
SEARCH_APPLICATION_TABLE,
|
||||
column_names,
|
||||
values_tuple,
|
||||
)
|
||||
return search_application
|
||||
|
||||
def update(self, engine_id: str, search_application: SearchApplication):
|
||||
"""
|
||||
Updates an existing Search Application configuration in BigQuery.
|
||||
|
||||
Identifies the row to update using the provided engine_id.
|
||||
|
||||
Args:
|
||||
engine_id: The engine_id of the Search Application configuration
|
||||
to update.
|
||||
search_application: A SearchApplication object containing the
|
||||
updated details (engine_id and region).
|
||||
|
||||
Returns:
|
||||
The updated SearchApplication object.
|
||||
|
||||
Raises:
|
||||
# Note: The underlying repository might raise exceptions on failure,
|
||||
# which are not explicitly handled here.
|
||||
# Consider adding error handling.
|
||||
"""
|
||||
update_dict = {
|
||||
"engine_id": f'"{search_application.engine_id}"',
|
||||
"region": f'"{search_application.region}"',
|
||||
}
|
||||
self.repository.update_row_by_id(
|
||||
SEARCH_APPLICATION_TABLE,
|
||||
SEARCH_APPLICATION_TABLE_ID_COLUMN,
|
||||
engine_id,
|
||||
update_dict,
|
||||
)
|
||||
Reference in New Issue
Block a user