chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 13:30:30 +08:00
commit 914fea506e
2793 changed files with 802106 additions and 0 deletions
@@ -0,0 +1,20 @@
# This file specifies files that are *not* uploaded to Google Cloud
# using gcloud. It follows the same syntax as .gitignore, with the addition of
# "#!include" directives (which insert the entries of the given .gitignore-style
# file at that point).
#
# For more information, run:
# $ gcloud topic gcloudignore
#
.gcloudignore
# If you would like to upload your .git directory, .gitignore file or files
# from your .gitignore file, remove the corresponding line
# below:
.git
.gitignore
# Python pycache:
__pycache__/
# Ignored by the build system
/setup.cfg
my_env
@@ -0,0 +1,7 @@
__pycache__/
.idea/
.vscode/
.venv/
env
bin
lib
@@ -0,0 +1,28 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM python:3.11-alpine
COPY requirements.txt /tmp
RUN pip install -r /tmp/requirements.txt
WORKDIR /app
COPY . /app/
ENV ENVIRONMENT="development"
ENV BIG_QUERY_DATASET=""
ENV FRONTEND_URL=""
EXPOSE 8080
ENTRYPOINT ["gunicorn", "main:app", "--workers=4", "--worker-class=uvicorn.workers.UvicornWorker", "--timeout=36000", "--bind=0.0.0.0:8080"]
@@ -0,0 +1,92 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# syntax=docker/dockerfile:1.4
FROM python:3.11 AS builder
# Build arguments
ARG IS_FIRST_DEPLOYMENT="False"
ARG GCLOUD_PROJECT=""
ARG ENVIRONMENT="development"
ARG FRONTEND_URL=""
ARG BIG_QUERY_DATASET=""
ARG VERTEX_AI_LOCATION=""
ARG VERTEX_AI_DATASTORE_ID=""
ARG VERTEX_AI_ENGINE_ID=""
# Environment variables for the build process
ENV IS_FIRST_DEPLOYMENT=$IS_FIRST_DEPLOYMENT
ENV GCLOUD_PROJECT=$GCLOUD_PROJECT
ENV ENVIRONMENT=$ENVIRONMENT
ENV FRONTEND_URL=$FRONTEND_URL
ENV BIG_QUERY_DATASET=$BIG_QUERY_DATASET
ENV VERTEX_AI_LOCATION=$VERTEX_AI_LOCATION
ENV VERTEX_AI_DATASTORE_ID=$VERTEX_AI_DATASTORE_ID
ENV VERTEX_AI_ENGINE_ID=$VERTEX_AI_ENGINE_ID
WORKDIR /code
COPY . .
COPY setup.py ./setup.py
RUN pip install --no-cache-dir -r /code/requirements.txt
RUN --mount=type=secret,id=gcp_credentials,target=/tmp/gcp_adc.json,required=true \
if [ "$IS_FIRST_DEPLOYMENT" = "True" ]; then \
echo "--- Running first deployment setup (IS_FIRST_DEPLOYMENT=True) ---"; \
echo "Using GCLOUD_PROJECT=${GCLOUD_PROJECT} for setup"; \
\
if [ ! -f /tmp/gcp_adc.json ]; then \
echo "CRITICAL ERROR: GCP credentials secret was expected but not mounted to /tmp/gcp_adc.json" >&2; \
exit 1; \
fi; \
echo "DEBUG: GCP credentials secret successfully mounted to /tmp/gcp_adc.json"; \
\
# Set GOOGLE_APPLICATION_CREDENTIALS specifically for this RUN command's execution context
export GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcp_adc.json; \
echo "DEBUG: GOOGLE_APPLICATION_CREDENTIALS for setup is set to: $GOOGLE_APPLICATION_CREDENTIALS"; \
\
echo "Executing setup.py ..."; \
# Now setup.py can make requests to GCP with provided credentials
python /code/setup.py || { echo "setup.py with failed!"; exit 1; }; \
echo "--- First deployment setup finished ---"; \
else \
echo "--- IS_FIRST_DEPLOYMENT is False, skipping first deployment setup. ---"; \
fi
FROM python:3.11-slim AS final
ARG GCLOUD_PROJECT=""
ARG GOOGLE_APPLICATION_CREDENTIALS=""
ARG ENVIRONMENT="development"
ARG FRONTEND_URL=""
ARG BIG_QUERY_DATASET=""
# Environment variables for the build process
ENV GCLOUD_PROJECT=$GCLOUD_PROJECT
ENV GOOGLE_APPLICATION_CREDENTIALS="/root/.config/gcloud/application_default_credentials.json"
ENV ENVIRONMENT=$ENVIRONMENT
ENV FRONTEND_URL=$FRONTEND_URL
ENV BIG_QUERY_DATASET=$BIG_QUERY_DATASET
WORKDIR /app
COPY --from=builder /code/requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r /app/requirements.txt
COPY --from=builder /code/main.py /app/main.py
COPY --from=builder /code/src /app/src
EXPOSE 8080
ENTRYPOINT ["gunicorn", "main:app", "--workers=4", "--worker-class=uvicorn.workers.UvicornWorker", "--timeout=36000", "--bind=0.0.0.0:8080"]
@@ -0,0 +1,132 @@
# QuickBotApp document-search-using-agent-builder
## Setting up
### 1. Create virtualenv and install dependencies
Create a virtual environment on the root of the application, activate it and install the requirements
```
# check if you are already in the env
pip -V
# if not then
python3 -m venv .venv
source .venv/bin/activate
pip3 install -r requirements.txt
```
> **IMPORTANT!** VS Code may not recognize your env, in that case type "ctrl + shift + P", then select "Python: Select Interpreter" and then select "Enter interpreter path..." and then select your .venv python interpreter, in this case .backend/.venv/bin/python
### 2. Setup gcloud credentials
```
gcloud auth list
gcloud config list
gcloud auth login
gcloud config set project <your project id>
gcloud auth application-default set-quota-project <your project id>
gcloud auth list
gcloud config list
```
### 3. Add environment variables
#### If you have Mac or Windows (or if you are using zsh console on Linux)
```
. ./local.env
```
#### If you have Linux
Open the file .venv/bin/activate and paste the env variables from `.local.env` after the PATH export, like this:
```
...
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/bin:$PATH"
export PATH
# Quickbot env variables
export ENVIRONMENT=development
export FRONTEND_URL=http://localhost:4200
export BIG_QUERY_DATASET=eren
...
```
Check that the env variables has been taken into account, running:
```
env
```
You should see the new env variables set there
### 4. Running the set up script
```
python3 setup.py
```
### 5. Run the application
Finally run using uvicorn
```
uvicorn main:app --reload --port 8080
```
## Code Styling & Commit Guidelines
To maintain code quality and consistency:
* **TypeScript (Frontend):** We follow [Angular Coding Style Guide](https://angular.dev/style-guide) by leveraging the use of [Google's TypeScript Style Guide](https://github.com/google/gts) using `gts`. This includes a formatter, linter, and automatic code fixer.
* **Python (Backend):** We adhere to the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html), using tools like `pylint` and `black` for linting and formatting.
* **Commit Messages:** We suggest following [Angular's Commit Message Guidelines](https://github.com/angular/angular/blob/main/contributing-docs/commit-message-guidelines.md) to create clear and descriptive commit messages.
#### Frontend (TypeScript with `gts`)
1. **Initialize `gts` (if not already done in the project):**
Navigate to the `frontend/` directory and run:
```bash
npx gts init
```
This will set up `gts` and create necessary configuration files (like `tsconfig.json`). Ensure your `tsconfig.json` (or a related gts config file like `.gtsrc`) includes an extension for `gts` defaults, typically:
```json
{
"extends": "./node_modules/gts/tsconfig-google.json",
// ... other configurations
}
```
2. **Check for linting issues:**
```bash
npm run lint
```
(This assumes a `lint` script is defined in `package.json`, e.g., `"lint": "gts lint"`)
3. **Fix linting issues automatically (where possible):**
```bash
npm run fix
```
(This assumes a `fix` script is defined in `package.json`, e.g., `"fix": "gts fix"`)
#### Backend (Python with `pylint` and `black`)
1. **Ensure Dependencies are Installed:**
Add `pylint` and `black` to your `backend/requirements.txt` file:
```
pylint
black
```
Then install them within your virtual environment:
```bash
pip install pylint black
# or pip install -r requirements.txt
```
2. **Configure `pylint`:**
It's recommended to have a `.pylintrc` file in your `backend/` directory to configure `pylint` rules. You might need to copy a standard one or generate one (`pylint --generate-rcfile > .pylintrc`).
3. **Check for linting issues with `pylint`:**
Navigate to the `backend/` directory and run:
```bash
pylint .
```
(Or specify modules/packages: `pylint your_module_name`)
4. **Format code with `black`:**
To automatically format all Python files in the current directory and subdirectories:
```bash
python -m black . --line-length=80
```
@@ -0,0 +1,14 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
@@ -0,0 +1,3 @@
export ENVIRONMENT=development
export FRONTEND_URL=http://localhost:4200
export BIG_QUERY_DATASET=eren
@@ -0,0 +1,104 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Main FastAPI application entry point for the Quick Bot Backend.
This module initializes the FastAPI application, configures CORS based on
the environment, defines root and version endpoints, includes API routers
(e.g., for search functionality), and provides an endpoint for audio
transcription using Google Cloud Speech-to-Text.
"""
from fastapi import FastAPI, File, UploadFile
from fastapi.middleware.cors import CORSMiddleware
from src.controller.search import router as search_router
from google.cloud import speech
from os import getenv
app = FastAPI()
def configure_cors(app):
"""Configures CORS middleware based on the environment."""
environment = getenv("ENVIRONMENT")
allowed_origins = []
if environment == "production":
frontend_url = getenv("FRONTEND_URL")
if not frontend_url:
raise ValueError(
"FRONTEND_URL environment variable not set in production"
)
allowed_origins.append(frontend_url)
elif environment == "development":
allowed_origins.append("*") # Allow all origins in development
else:
raise ValueError(
f"""Invalid ENVIRONMENT: {environment}.
Must be 'production' or 'development'"""
)
app.add_middleware(
CORSMiddleware,
allow_origins=allowed_origins,
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# Create a route to handle GET requests on root
@app.get("/")
async def root():
return "You are calling Quick Bot Backend"
# Create a route to handle GET requests on /version
@app.get("/api/version")
def version():
return "v0.0.1"
@app.post("/api/audio_chat")
async def audio_chat(audio_file: UploadFile = File(...)):
client = speech.SpeechClient()
audio_content = await audio_file.read()
audio = speech.RecognitionAudio(content=audio_content)
config = speech.RecognitionConfig(
language_code="en-US",
sample_rate_hertz=48000,
model="default",
audio_channel_count=1,
enable_word_confidence=True,
enable_word_time_offsets=True,
)
operation = client.long_running_recognize(config=config, audio=audio)
print("Waiting for operation to complete...")
response = operation.result(timeout=90)
print(response)
text = ""
for result in response.results:
print(f"Transcript: {result.alternatives[0].transcript}")
text = result.alternatives[0].transcript
return text, 200
configure_cors(app)
app.include_router(search_router)
@@ -0,0 +1,399 @@
# This Pylint rcfile contains a best-effort configuration to uphold the
# best-practices and style described in the Google Python style guide:
# https://google.github.io/styleguide/pyguide.html
#
# Its canonical open-source location is:
# https://google.github.io/styleguide/pylintrc
[MAIN]
# Files or directories to be skipped. They should be base names, not paths.
ignore=third_party
# Files or directories matching the regex patterns are skipped. The regex
# matches against base names, not paths.
ignore-patterns=
# Pickle collected data for later comparisons.
persistent=no
# List of plugins (as comma separated values of python modules names) to load,
# usually to register additional checkers.
load-plugins=
# Use multiple processes to speed up Pylint.
jobs=4
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
unsafe-load-any-extension=no
[MESSAGES CONTROL]
# Only show warnings with the listed confidence levels. Leave empty to show
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
confidence=
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
# multiple time (only on the command line, not in the configuration file where
# it should appear only once). See also the "--disable" option for examples.
#enable=
# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifiers separated by comma (,) or put this
# option multiple times (only on the command line, not in the configuration
# file where it should appear only once).You can also use "--disable=all" to
# disable everything first and then reenable specific checks. For example, if
# you want to run only the similarities checker, you can use "--disable=all
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
disable=R,
abstract-method,
apply-builtin,
arguments-differ,
attribute-defined-outside-init,
backtick,
bad-option-value,
basestring-builtin,
buffer-builtin,
c-extension-no-member,
consider-using-enumerate,
cmp-builtin,
cmp-method,
coerce-builtin,
coerce-method,
delslice-method,
div-method,
eq-without-hash,
execfile-builtin,
file-builtin,
filter-builtin-not-iterating,
fixme,
getslice-method,
global-statement,
hex-method,
idiv-method,
implicit-str-concat,
import-error,
import-self,
import-star-module-level,
input-builtin,
intern-builtin,
invalid-str-codec,
locally-disabled,
long-builtin,
long-suffix,
map-builtin-not-iterating,
misplaced-comparison-constant,
missing-function-docstring,
metaclass-assignment,
next-method-called,
next-method-defined,
no-absolute-import,
no-init, # added
no-member,
no-name-in-module,
no-self-use,
nonzero-method,
oct-method,
old-division,
old-ne-operator,
old-octal-literal,
old-raise-syntax,
parameter-unpacking,
print-statement,
raising-string,
range-builtin-not-iterating,
raw_input-builtin,
rdiv-method,
reduce-builtin,
relative-import,
reload-builtin,
round-builtin,
setslice-method,
signature-differs,
standarderror-builtin,
suppressed-message,
sys-max-int,
trailing-newlines,
unichr-builtin,
unicode-builtin,
unnecessary-pass,
unpacking-in-except,
useless-else-on-loop,
useless-suppression,
using-cmp-argument,
wrong-import-order,
xrange-builtin,
zip-builtin-not-iterating,
[REPORTS]
# Set the output format. Available formats are text, parseable, colorized, msvs
# (visual studio) and html. You can also give a reporter class, eg
# mypackage.mymodule.MyReporterClass.
output-format=text
# Tells whether to display a full report or only the messages
reports=no
# Python expression which should return a note less than 10 (10 is the highest
# note). You have access to the variables errors warning, statement which
# respectively contain the number of errors / warnings messages and the total
# number of statements analyzed. This is used by the global evaluation report
# (RP0004).
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
# Template used to display messages. This is a python new-style format string
# used to format the message information. See doc for all details
#msg-template=
[BASIC]
# Good variable names which should always be accepted, separated by a comma
good-names=main,_
# Bad variable names which should always be refused, separated by a comma
bad-names=
# Colon-delimited sets of names that determine each other's naming style when
# the name regexes allow several styles.
name-group=
# Include a hint for the correct naming format with invalid-name
include-naming-hint=no
# List of decorators that produce properties, such as abc.abstractproperty. Add
# to this list to register other decorators that produce valid properties.
property-classes=abc.abstractproperty,cached_property.cached_property,cached_property.threaded_cached_property,cached_property.cached_property_with_ttl,cached_property.threaded_cached_property_with_ttl
# Regular expression matching correct function names
function-rgx=^(?:(?P<exempt>setUp|tearDown|setUpModule|tearDownModule)|(?P<camel_case>_?[A-Z][a-zA-Z0-9]*)|(?P<snake_case>_?[a-z][a-z0-9_]*))$
# Regular expression matching correct variable names
variable-rgx=^[a-z][a-z0-9_]*$
# Regular expression matching correct constant names
const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
# Regular expression matching correct attribute names
attr-rgx=^_{0,2}[a-z][a-z0-9_]*$
# Regular expression matching correct argument names
argument-rgx=^[a-z][a-z0-9_]*$
# Regular expression matching correct class attribute names
class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
# Regular expression matching correct inline iteration names
inlinevar-rgx=^[a-z][a-z0-9_]*$
# Regular expression matching correct class names
class-rgx=^_?[A-Z][a-zA-Z0-9]*$
# Regular expression matching correct module names
module-rgx=^(_?[a-z][a-z0-9_]*|__init__)$
# Regular expression matching correct method names
method-rgx=(?x)^(?:(?P<exempt>_[a-z0-9_]+__|runTest|setUp|tearDown|setUpTestCase|tearDownTestCase|setupSelf|tearDownClass|setUpClass|(test|assert)_*[A-Z0-9][a-zA-Z0-9_]*|next)|(?P<camel_case>_{0,2}[A-Z][a-zA-Z0-9_]*)|(?P<snake_case>_{0,2}[a-z][a-z0-9_]*))$
# Regular expression which should only match function or class names that do
# not require a docstring.
no-docstring-rgx=(__.*__|main|test.*|.*test|.*Test)$
# Minimum line length for functions/classes that require docstrings, shorter
# ones are exempt.
docstring-min-length=12
[TYPECHECK]
# List of decorators that produce context managers, such as
# contextlib.contextmanager. Add to this list to register other decorators that
# produce valid context managers.
contextmanager-decorators=contextlib.contextmanager,contextlib2.contextmanager
# List of module names for which member attributes should not be checked
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=
# List of class names for which member attributes should not be checked (useful
# for classes with dynamically set attributes). This supports the use of
# qualified names.
ignored-classes=optparse.Values,thread._local,_thread._local
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E1101 when accessed. Python regular
# expressions are accepted.
generated-members=
[FORMAT]
# Maximum number of characters on a single line.
max-line-length=80
# TODO(https://github.com/pylint-dev/pylint/issues/3352): Direct pylint to exempt
# lines made too long by directives to pytype.
# Regexp for a line that is allowed to be longer than the limit.
ignore-long-lines=(?x)(
^\s*(\#\ )?<?https?://\S+>?$|
^\s*(from\s+\S+\s+)?import\s+.+$)
# Allow the body of an if to be on the same line as the test if there is no
# else.
single-line-if-stmt=yes
# Maximum number of lines in a module
max-module-lines=99999
# String used as indentation unit. The internal Google style guide mandates 2
# spaces. Google's externaly-published style guide says 4, consistent with
# PEP 8. Here, we use 2 spaces, for conformity with many open-sourced Google
# projects (like TensorFlow).
indent-string=' '
# Number of spaces of indent required inside a hanging or continued line.
indent-after-paren=4
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
expected-line-ending-format=
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
notes=TODO
[STRING]
# This flag controls whether inconsistent-quotes generates a warning when the
# character used as a quote delimiter is used inconsistently within a module.
check-quote-consistency=yes
[VARIABLES]
# Tells whether we should check for unused import in __init__ files.
init-import=no
# A regular expression matching the name of dummy variables (i.e. expectedly
# not used).
dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_)
# List of additional names supposed to be defined in builtins. Remember that
# you should avoid to define new builtins when possible.
additional-builtins=
# List of strings which can identify a callback function by name. A callback
# name must start or end with one of those strings.
callbacks=cb_,_cb
# List of qualified module names which can have objects that can redefine
# builtins.
redefining-builtins-modules=six,six.moves,past.builtins,future.builtins,functools
[LOGGING]
# Logging modules to check that the string format arguments are in logging
# function parameter format
logging-modules=logging,absl.logging,tensorflow.io.logging
[SIMILARITIES]
# Minimum lines number of a similarity.
min-similarity-lines=4
# Ignore comments when computing similarities.
ignore-comments=yes
# Ignore docstrings when computing similarities.
ignore-docstrings=yes
# Ignore imports when computing similarities.
ignore-imports=no
[SPELLING]
# Spelling dictionary name. Available dictionaries: none. To make it working
# install python-enchant package.
spelling-dict=
# List of comma separated words that should not be checked.
spelling-ignore-words=
# A path to a file that contains private dictionary; one word per line.
spelling-private-dict-file=
# Tells whether to store unknown words to indicated private dictionary in
# --spelling-private-dict-file option instead of raising a message.
spelling-store-unknown-words=no
[IMPORTS]
# Deprecated modules which should not be used, separated by a comma
deprecated-modules=regsub,
TERMIOS,
Bastion,
rexec,
sets
# Create a graph of every (i.e. internal and external) dependencies in the
# given file (report RP0402 must not be disabled)
import-graph=
# Create a graph of external dependencies in the given file (report RP0402 must
# not be disabled)
ext-import-graph=
# Create a graph of internal dependencies in the given file (report RP0402 must
# not be disabled)
int-import-graph=
# Force import order to recognize a module as part of the standard
# compatibility libraries.
known-standard-library=
# Force import order to recognize a module as part of a third party library.
known-third-party=enchant, absl
# Analyse import fallback blocks. This can be used to support both Python 2 and
# 3 compatible code, which means that the block might have code that exists
# only in one or another interpreter, leading to false positives when analysed.
analyse-fallback-blocks=no
[CLASSES]
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,
__new__,
setUp
# List of member names, which should be excluded from the protected access
# warning.
exclude-protected=_asdict,
_fields,
_replace,
_source,
_make
# List of valid names for the first argument in a class method.
valid-classmethod-first-arg=cls,
class_
# List of valid names for the first argument in a metaclass class method.
valid-metaclass-classmethod-first-arg=mcs
@@ -0,0 +1,30 @@
gunicorn==20.1.0; python_version > '3.0'
gunicorn==19.10.0; python_version < '3.0'
uvicorn~=0.17.0
fastapi~=0.111.1
fastapi-utilities==0.2.0
cloudpickle==2.2.1
grpcio==1.66.2
urllib3==1.26.16
typing-inspect==0.9.0
typing_extensions==4.12.2
pydantic==2.9.2
requests==2.31.0
langchain==0.3.2
langchain-community==0.3.1
langchain-core==0.3.9
langchain-google-vertexai==2.0.4
google-cloud-aiplatform==1.69.0
google-cloud-bigquery==3.26.0
google-cloud-tasks==2.16.5
google-cloud-logging==3.11.2
google-cloud-speech==2.27.0
google-cloud-discoveryengine==0.13.4
pylint
black
pytest
pytest-cov
pytest-watch
@@ -0,0 +1,14 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
@@ -0,0 +1,69 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utility functions for setting up Google BigQuery datasets and tables."""
from typing import List
from google.cloud.bigquery import Client as BigQueryClient, Table, SchemaField, TableReference
bigquery_client = BigQueryClient()
PROJECT_ID = bigquery_client.project
def create_dataset(dataset_name: str):
"""Deletes an existing BigQuery dataset and recreates it.
If the dataset already exists, it and all its contents
will be deleted first.
Then, a new empty dataset with the specified name is created.
Args:
dataset_name: The name for the BigQuery dataset.
"""
dataset_id = f"{PROJECT_ID}.{dataset_name}"
try:
bigquery_client.delete_dataset(
dataset_id, delete_contents=True, not_found_ok=True
)
print(f"Dataset {dataset_id} deleted (if it existed).")
except Exception as e:
print(f"Error deleting dataset {dataset_id}: {e}")
print(f"Creating dataset {dataset_id}...")
bigquery_client.create_dataset(dataset_name, exists_ok=True) # exists_ok=True in case delete failed but it exists
print(f"Dataset {dataset_id} ensured.")
def create_table(dataset: str, table_name: str, schema: List[SchemaField]):
"""Creates a BigQuery table within a specified dataset.
If the table already exists, it will be deleted and recreated.
Args:
dataset: The name of the dataset where the table will be created.
table_name: The name for the new BigQuery table.
schema: A list of SchemaField objects defining the table's structure.
"""
table_id_full = f"{PROJECT_ID}.{dataset}.{table_name}"
table_ref = TableReference.from_string(table_id_full)
try:
bigquery_client.delete_table(table_ref, not_found_ok=True)
print(f"Table {table_id_full} deleted (if it existed).")
except Exception as e:
print(f"Notice: Could not delete table {table_id_full} (may not exist or other issue): {e}")
print(f"Creating table {table_id_full}...")
bq_table = Table(table_ref, schema=schema)
bigquery_client.create_table(bq_table)
print(f"Table {table_id_full} created successfully.")
@@ -0,0 +1,143 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from google.cloud import discoveryengine_v1 as discoveryengine
from google.api_core.exceptions import AlreadyExists, NotFound
def create_vertex_ai_datastore(
project_id: str,
location: str,
datastore_id: str,
datastore_display_name: str,
):
"""Creates a Vertex AI Search Datastore if it doesn't exist."""
client = discoveryengine.DataStoreServiceClient()
parent = client.collection_path(project_id, location, "default_collection")
datastore_name_full = client.data_store_path(project_id, location, datastore_id)
try:
ds = client.get_data_store(name=datastore_name_full)
print(f"Datastore '{datastore_id}' already exists in location '{location}': {ds.name}")
return ds
except NotFound:
print(f"Datastore '{datastore_id}' not found in location '{location}'. Attempting to create...")
datastore = discoveryengine.DataStore(
display_name=datastore_display_name,
industry_vertical=discoveryengine.IndustryVertical.GENERIC,
content_config=discoveryengine.DataStore.ContentConfig.CONTENT_REQUIRED,
)
try:
operation = client.create_data_store(
parent=parent,
data_store=datastore,
data_store_id=datastore_id,
)
print(f"Waiting for Datastore '{datastore_id}' creation (LRO: {operation.operation.name})...")
created_datastore = operation.result(timeout=300) # 5 minutes timeout
print(f"Successfully created Datastore: {created_datastore.name}")
return created_datastore
except AlreadyExists:
print(f"Datastore '{datastore_id}' creation reported AlreadyExists. Fetching existing.")
return client.get_data_store(name=datastore_name_full)
except Exception as e:
print(f"Error creating Datastore '{datastore_id}': {e}")
raise
def import_documents_to_datastore(
project_id: str,
location: str,
datastore_id: str,
gcs_uri: str,
):
"""Imports documents from GCS into the specified Datastore."""
client = discoveryengine.DocumentServiceClient()
parent_branch = client.branch_path(
project=project_id,
location=location,
data_store=datastore_id,
branch="default_branch",
)
request = discoveryengine.ImportDocumentsRequest(
parent=parent_branch,
gcs_source=discoveryengine.GcsSource(input_uris=[gcs_uri], data_schema="content"),
reconciliation_mode=discoveryengine.ImportDocumentsRequest.ReconciliationMode.INCREMENTAL
)
try:
print(f"Starting document import from '{gcs_uri}' into Datastore '{datastore_id}'...")
operation = client.import_documents(request=request)
print(f"Waiting for document import to complete (LRO: {operation.operation.name}). This may take several minutes...")
response = operation.result(timeout=1800) # 30 minutes timeout
if response.error_samples and len(response.error_samples) > 0:
print(f"Document import completed with errors. Error Config {response.error_config}")
for i, error_sample in enumerate(response.error_samples):
print(f" Error sample {i+1}: {error_sample.message}")
raise Exception("Document import failed with errors", response)
else:
print(f"Successfully imported documents.")
return response
except Exception as e:
print(f"Error during document import for Datastore '{datastore_id}': {e}")
raise
def create_vertex_ai_engine(
project_id: str,
location: str,
engine_id: str,
engine_display_name: str,
datastore_ids_list: list[str],
):
"""Creates a Vertex AI Search Engine (App) if it doesn't exist."""
client = discoveryengine.EngineServiceClient()
parent_collection = client.collection_path(project_id, location, "default_collection")
engine_name_full = client.engine_path(project_id, location, "default_collection", engine_id)
try:
eng = client.get_engine(name=engine_name_full)
print(f"Engine '{engine_id}' already exists in location '{location}': {eng.name}")
return eng
except NotFound:
print(f"Engine '{engine_id}' not found in location '{location}'. Attempting to create...")
engine_config = discoveryengine.Engine(
display_name=engine_display_name,
solution_type=discoveryengine.SolutionType.SOLUTION_TYPE_SEARCH,
data_store_ids=datastore_ids_list,
common_config=discoveryengine.Engine.CommonConfig(company_name="QuickBot App"),
search_engine_config=discoveryengine.Engine.SearchEngineConfig(
search_tier="SEARCH_TIER_STANDARD",
search_add_ons=["SEARCH_ADD_ON_LLM"]
)
)
try:
operation = client.create_engine(
parent=parent_collection,
engine=engine_config,
engine_id=engine_id,
)
print(f"Waiting for Engine '{engine_id}' creation (LRO: {operation.operation.name})...")
created_engine = operation.result(timeout=600) # 10 minutes timeout
print(f"Successfully created Engine: {created_engine.name}")
return created_engine
except AlreadyExists:
print(f"Engine '{engine_id}' creation reported AlreadyExists. Fetching existing.")
return client.get_engine(name=engine_name_full)
except Exception as e:
print(f"Error creating Engine '{engine_id}': {e}")
raise
@@ -0,0 +1,120 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Initializes the BigQuery and Vertex AI Search environment for the application.
This script performs the following actions:
1. Retrieves configuration from environment variables or uses defaults.
2. Creates the specified BigQuery dataset if it doesn't already exist.
3. Creates the 'search_applications' table within that dataset.
4. Creates a Vertex AI Search Datastore if it doesn't already exist.
5. Imports documents from a specified GCS bucket into the Datastore.
6. Creates a Vertex AI Search Engine (App) linked to the Datastore.
Usage:
Run this script directly (e.g., `python setup.py`).
Set environment variables to override defaults:
- 'BIG_QUERY_DATASET'
- 'GOOGLE_CLOUD_PROJECT'
- 'VERTEX_AI_SEARCH_LOCATION'
- 'VERTEX_AI_DATASTORE_ID'
- 'VERTEX_AI_ENGINE_ID'
"""
from os import getenv
from scripts.big_query_setup import create_dataset, create_table
from src.service.search_application import SEARCH_APPLICATION_TABLE
from src.model.search import SearchApplication
from scripts.vertexai_search_setup import create_vertex_ai_datastore, create_vertex_ai_engine, import_documents_to_datastore
def main():
# 1. BigQuery Setup
print("--- Setting up BigQuery ---")
BIG_QUERY_DATASET = getenv("BIG_QUERY_DATASET", "quickbot_default_bq_dataset")
GCLOUD_PROJECT = getenv("GCLOUD_PROJECT", "my-gcloud-project")
create_dataset(BIG_QUERY_DATASET)
create_table(
BIG_QUERY_DATASET, SEARCH_APPLICATION_TABLE, SearchApplication.__schema__()
)
# 2. Vertex AI Search Setup
print("--- Setting up Vertex AI Search ---")
VERTEX_AI_LOCATION = getenv("VERTEX_AI_LOCATION", "global")
VERTEX_AI_DATASTORE_ID = getenv("VERTEX_AI_DATASTORE_ID", "quickbot_alphabet_pdfs_ds")
VERTEX_AI_ENGINE_ID = getenv("VERTEX_AI_ENGINE_ID", "quickbot_alphabet_search_engine")
GCS_SOURCE_URI = "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/*.pdf"
DATASTORE_DISPLAY_NAME_PREFIX = "Alphabet Investor Docs DS"
ENGINE_DISPLAY_NAME_PREFIX = "Alphabet Investor Engine"
datastore_display_name = f"{DATASTORE_DISPLAY_NAME_PREFIX} ({VERTEX_AI_DATASTORE_ID})"
engine_display_name = f"{ENGINE_DISPLAY_NAME_PREFIX} ({VERTEX_AI_ENGINE_ID})"
try:
# Create/Get Datastore
print(f"Attempting to create/get Datastore '{VERTEX_AI_DATASTORE_ID}' in project '{GCLOUD_PROJECT}' location '{VERTEX_AI_LOCATION}'...")
datastore = create_vertex_ai_datastore(
GCLOUD_PROJECT, VERTEX_AI_LOCATION, VERTEX_AI_DATASTORE_ID, datastore_display_name
)
if not datastore:
print("Datastore creation/retrieval failed. Aborting further Vertex AI Search setup.")
print("--- Application setup finished (with errors) ---")
raise
print(f"Successfully ensured Datastore exists: {datastore.name}")
# Import documents into Datastore
# Note: This will attempt to import documents every time the script runs.
# For production, you might want to add a check to skip this if documents
# are already present or if a previous import was successful.
print(f"\nAttempting to import documents from '{GCS_SOURCE_URI}' into datastore: {datastore.name}")
import_documents_to_datastore(
GCLOUD_PROJECT, VERTEX_AI_LOCATION, VERTEX_AI_DATASTORE_ID, GCS_SOURCE_URI
)
# Note: Document import can take a long time. The script waits.
print("Document import process initiated/completed.\n")
# Create/Get Engine
print(f"Attempting to create/get Engine '{VERTEX_AI_ENGINE_ID}' in project '{GCLOUD_PROJECT}' location '{VERTEX_AI_LOCATION}'...")
# The create_vertex_ai_engine function expects a list of datastore IDs (not full resource names).
engine = create_vertex_ai_engine(
GCLOUD_PROJECT,
VERTEX_AI_LOCATION,
VERTEX_AI_ENGINE_ID,
engine_display_name,
[VERTEX_AI_DATASTORE_ID] # Pass the Datastore ID string
)
if not engine:
print("Engine creation/retrieval failed.")
print("--- Application setup finished (with errors) ---")
raise
print(f"Successfully ensured Engine exists: {engine.name}")
print("\nVertex AI Search setup completed successfully.")
except Exception as e:
print(f"A critical error occurred during the setup process: {e}")
import traceback
print("Detailed traceback:")
print(traceback.format_exc())
# If running in Docker build, exiting with non-zero will fail the build
import sys
sys.exit(1)
print("\n--- Application setup finished ---")
print("\nSuccess! All resources should now be configured.\n")
if __name__ == "__main__":
main()
@@ -0,0 +1,14 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
@@ -0,0 +1,14 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
@@ -0,0 +1,153 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""API endpoints for managing and performing document searches."""
from fastapi import APIRouter, Request, HTTPException, Response
from google.cloud import storage
from pydantic import BaseModel
from src.model.http_status import BadRequest
from src.model.search import CreateSearchRequest, SearchApplication
from src.service.engine import EngineService
from src.service.search import SearchService
from src.service.search_application import SearchApplicationService
class SignedUrlRequest(BaseModel):
"""Request model for fetching a document directly from GCS."""
gcs_url: str
storage_client = storage.Client()
router = APIRouter(
prefix="/api/search",
tags=["searches"],
responses={404: {"description": "Not found"}},
)
@router.post("")
async def search(item: CreateSearchRequest):
"""
Performs a search using the configured Search Application.
Args:
item: The search request containing the search term.
Raises:
BadRequest: If no Search Application is configured for the project.
Returns:
The search results from the SearchService.
"""
service = SearchApplicationService()
search_application = service.get()
if not search_application:
raise BadRequest(detail="No Search Application found on project")
service = SearchService(
search_application,
)
return service.search(item.term)
@router.get("/engines")
async def get_all_engines():
"""Retrieves all available Search Engines."""
service = EngineService()
return service.get_all()
@router.get("/application")
async def get_search_application():
"""Retrieves the currently configured Search Application."""
service = SearchApplicationService()
return service.get()
@router.post("/application")
async def create_search_application(search_application: SearchApplication):
"""
Creates a new Search Application configuration.
Args:
search_application: The details of the Search Application to create.
Returns:
The created Search Application configuration.
"""
service = SearchApplicationService()
return service.create(search_application)
@router.put("/application/{engine_id}")
async def update_search_application(
engine_id: str, search_application: SearchApplication
):
"""
Updates an existing Search Application configuration.
Args:
engine_id: The ID of the engine associated with
the application to update.
search_application: The updated details for the Search Application.
Returns:
The updated Search Application configuration.
"""
service = SearchApplicationService()
return service.update(engine_id, search_application)
@router.post("/doc")
async def get_document(request: Request, response_model=None):
"""
Fetches a document directly from GCS and returns its content.
Expects a JSON body with a 'gcs_url' field specifying the full GCS path
(e.g., "gs://your-bucket-name/your-file.pdf").
Args:
request: The incoming FastAPI request object.
Raises:
HTTPException: If the GCS URL is invalid or the file cannot be fetched.
Returns:
A FastAPI Response object containing the raw PDF content.
"""
try:
req_body = await request.json()
signed_url_request = SignedUrlRequest(**req_body)
gcs_url = signed_url_request.gcs_url
bucket_name = gcs_url.split("/")[2]
object_name = "/".join(gcs_url.split("/")[3:])
# Get a reference to the blob (PDF file)
bucket = storage_client.bucket(bucket_name)
blob = bucket.blob(object_name)
# Download the blob content as bytes
pdf_content = blob.download_as_bytes()
# Return the PDF content with appropriate headers
return Response(content=pdf_content, media_type="application/pdf")
except Exception as e:
raise HTTPException(
status_code=500, detail=f"Error fetching PDF: {str(e)}"
)
@@ -0,0 +1,14 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
@@ -0,0 +1,53 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Defines custom HTTP exception classes for the FastAPI application.
This module provides reusable exception classes that inherit from FastAPI's
HTTPException, allowing for standardized error responses with specific
status codes and detail messages.
"""
from fastapi import HTTPException
class ResourceAlreadyExists(HTTPException):
"""
Custom exception for HTTP 400 Bad Request errors.
Used specifically when an attempt is made to create a resource that
already exists (e.g., creating a duplicate configuration).
Defaults to status code 400.
"""
def __init__(self, detail="Resource already exists"):
"""Initializes the exception with a default detail message."""
super().__init__(status_code=400, detail=detail)
class BadRequest(HTTPException):
"""
Custom exception for general HTTP 400 Bad Request errors.
Can be used for various client-side errors like invalid input,
missing required data, or violating business logic rules before
processing. Defaults to status code 400.
Note: The default detail message "Resource already exists" seems
inconsistent with a general BadRequest. Consider changing it to
a more generic message like "Bad Request" or requiring a specific
detail message upon instantiation.
"""
def __init__(self, detail="Resource already exists"):
super().__init__(status_code=400, detail=detail)
@@ -0,0 +1,119 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Defines data models for search operations and configurations.
This module includes Pydantic models for API requests/responses related to
search applications and engines, as well as dataclasses for representing
search results internally. It also handles fetching the default Google Cloud
Project ID.
"""
from dataclasses import dataclass
from typing import List, Optional
from pydantic import BaseModel
from google.cloud.bigquery import SchemaField, Row
from google.api_core.client_options import ClientOptions
import google.auth
_, PROJECT_ID = google.auth.default()
class CreateSearchRequest(BaseModel):
"""Request model for initiating a search."""
term: str
class SearchApplication(BaseModel):
"""Represents the configuration for a Discovery Engine
Search Application."""
engine_id: str
region: str
@classmethod
def __schema__(cls) -> List[SchemaField]:
"""Defines the BigQuery schema for storing SearchApplication data."""
return [
SchemaField("engine_id", "STRING", mode="REQUIRED"),
SchemaField("region", "STRING", mode="REQUIRED"),
]
@classmethod
def from_row(cls, row: Row) -> 'SearchApplication':
"""Creates a SearchApplication instance from a BigQuery Row object.
Args:
row: The BigQuery Row object. Assumes row contains fields matching the schema.
Returns:
A SearchApplication instance.
"""
# Access by field name for robustness, assuming schema matches
return cls(engine_id=row["engine_id"], region=row["region"])
def to_dict(self):
"""Converts the SearchApplication instance to a dictionary."""
return {
"engine_id": self.engine_id,
"region": self.region,
}
def get_client_options(self) -> Optional[ClientOptions]:
"""Generates API client options based on the application's region.
Returns:
ClientOptions configured with the regional endpoint, or None if
the region is 'global'.
"""
return (
ClientOptions(
api_endpoint=f"{self.region}-discoveryengine.googleapis.com"
)
if self.region != "global"
else None
)
def get_serving_config(self) -> str:
"""Constructs the full serving config path for the Discovery Engine API.
Returns:
The formatted serving config string.
"""
serving_config = f"projects/{PROJECT_ID}/locations/{self.region}"
serving_config += f"/collections/default_collection/engines/{self.engine_id}"
serving_config += "/servingConfigs/default_config"
return serving_config
class Engine(BaseModel):
"""Represents a discovered Discovery Engine instance."""
name: str
engine_id: str
region: str
@dataclass
class SearchResult:
"""Represents a single document result from a search query."""
document_id: str
title: str
snippet: str
link: Optional[str] = None
content: Optional[str] = None
@dataclass
class SearchResultsWithSummary:
"""Represents the complete results of a search, including a summary."""
results: List[SearchResult]
summary: Optional[str] = None
@@ -0,0 +1,14 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
@@ -0,0 +1,318 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Provides a repository class for interacting with Google BigQuery.
This module defines the `BigQueryRepository` class, which encapsulates
common BigQuery operations like running queries, fetching rows by ID,
inserting, updating, and deleting rows. It relies on the 'BIG_QUERY_DATASET'
environment variable to target the correct dataset.
"""
import datetime
from decimal import Decimal
from os import getenv
from typing import Dict, List, Any, Optional, Tuple
from google.cloud.bigquery import (
Client,
QueryJobConfig,
ScalarQueryParameter,
ArrayQueryParameter,
TableReference,
DatasetReference
)
from google.cloud.bigquery.table import RowIterator
from google.cloud.exceptions import GoogleCloudError
_BIG_QUERY_DATASET_ENV = getenv("BIG_QUERY_DATASET")
if not _BIG_QUERY_DATASET_ENV:
raise ValueError(
"The BIG_QUERY_DATASET environment variable is not set. "
"This is required for the BigQueryRepository to function."
)
BIG_QUERY_DATASET: str = _BIG_QUERY_DATASET_ENV
class BigQueryRepository:
"""
A repository class for simplifying interactions with Google BigQuery,
using parameterized queries to prevent SQL injection.
"""
def __init__(self):
"""Initializes the BigQuery client and project ID."""
self.client: Client = Client()
self.project_id: str = self.client.project
self.dataset_id: str = BIG_QUERY_DATASET
def _get_table_ref(self, table_id: str) -> TableReference:
"""Helper to get a TableReference object."""
dataset_ref = DatasetReference(self.project_id, self.dataset_id)
return TableReference(dataset_ref, table_id)
def _validate_column_name(self, column_name: str):
"""Basic validation for column names."""
if not column_name.replace("_", "").isalnum():
raise ValueError(f"Invalid column name: {column_name}")
def _create_scalar_query_parameter(self, name: str, value: Any) -> ScalarQueryParameter:
"""
Creates a ScalarQueryParameter by inferring the BigQuery type from the value.
"""
if isinstance(value, bool):
return ScalarQueryParameter(name, "BOOL", value)
if isinstance(value, int):
return ScalarQueryParameter(name, "INT64", value)
if isinstance(value, float):
return ScalarQueryParameter(name, "FLOAT64", value)
if isinstance(value, str):
return ScalarQueryParameter(name, "STRING", value)
if isinstance(value, datetime.datetime):
return ScalarQueryParameter(name, "TIMESTAMP", value)
if isinstance(value, datetime.date):
return ScalarQueryParameter(name, "DATE", value)
if isinstance(value, Decimal):
return ScalarQueryParameter(name, "NUMERIC", value)
if value is None:
# For None, the type of the column in BQ will ultimately determine how NULL is handled.
# Using STRING type for the parameter is a common safe default.
return ScalarQueryParameter(name, "STRING", None) # Default to STRING for None
# Fallback: attempt to convert other types to string.
try:
str_value = str(value)
return ScalarQueryParameter(name, "STRING", str_value)
except Exception as e:
raise TypeError(
f"Value for param '{name}' of type {type(value)} "
f"could not be converted to a supported BigQuery scalar type: {e}"
)
def run_query(self, query: str, job_config: Optional[QueryJobConfig] = None) -> RowIterator:
"""
Executes a BigQuery SQL query and returns the results.
Args:
query: The SQL query string to execute.
job_config: Optional QueryJobConfig for parameterized queries.
Returns:
A RowIterator object to iterate over the query results.
Raises:
google.cloud.exceptions.GoogleCloudError: If the query fails.
"""
try:
query_job = self.client.query(query, job_config=job_config)
return query_job.result()
except GoogleCloudError as e:
print(f"BigQuery Error: {e}")
print(f"Query: {query}")
if job_config and job_config.query_parameters:
print(f"Parameters: {job_config.query_parameters}")
raise
def get_row_by_id(self, table_id: str, id_column: str, id_value: Any, id_value_type: str = "STRING") -> RowIterator:
"""
Retrieves a single row from a table based on its ID using parameterized query.
Args:
table_id: The ID of the table (without dataset prefix).
id_column: The name of the column containing the ID.
id_value: The specific ID value to search for.
id_value_type: The BigQuery type of the ID value (e.g., "STRING", "INT64").
Returns:
A RowIterator containing the matching row(s).
"""
self._validate_column_name(id_column)
query = f"""
SELECT * FROM `{self.project_id}.{self.dataset_id}.{table_id}`
WHERE `{id_column}` = @id_value;
"""
job_config = QueryJobConfig(
query_parameters=[
ScalarQueryParameter("id_value", id_value_type, id_value)
]
)
return self.run_query(query, job_config=job_config)
def insert_rows_json(self, table_id: str, rows: List[Dict[str, Any]]) -> List[Dict]:
"""
Inserts multiple rows into the specified table using JSON.
This is the recommended method for insertions due to its safety and efficiency.
Args:
table_id: The ID of the table.
rows: A list of dictionaries, where each dictionary represents a row
(column names as keys, values as row data).
Returns:
A list of error dictionaries if any rows failed to insert,
otherwise an empty list.
"""
table_ref = self._get_table_ref(table_id)
errors_from_client = self.client.insert_rows_json(table_ref, rows)
errors_list: List[Dict[str, Any]] = list(errors_from_client) # type: ignore
if errors_list:
print(f"Errors inserting rows into {table_id}: {errors_list}")
return errors_list
def insert_row(self, table_id: str, column_names: List[str], values: Tuple[Any, ...]) -> None:
"""
Inserts a new row into the specified table using a parameterized query.
Prefer `insert_rows_json` for multiple rows or complex data.
Args:
table_id: The ID of the table.
column_names: A list of column names for the insert.
values: A tuple of values corresponding to the column_names.
Raises:
ValueError: If column names are invalid or counts don't match.
TypeError: If a value in `values` cannot be mapped to a BigQuery scalar type.
"""
if not column_names or not values:
raise ValueError("Column names and values cannot be empty.")
if len(column_names) != len(values):
raise ValueError("Number of column names must match number of values.")
for col_name in column_names:
self._validate_column_name(col_name)
cols_str = ", ".join(f"`{col}`" for col in column_names)
placeholders = ", ".join([f"@param{i}" for i in range(len(values))])
query_params: List[ScalarQueryParameter] = []
for i, current_value in enumerate(values):
param_name = f"param{i}"
query_params.append(self._create_scalar_query_parameter(param_name, current_value))
query = f"""
INSERT INTO `{self.project_id}.{self.dataset_id}.{table_id}` ({cols_str})
VALUES ({placeholders});
"""
job_config = QueryJobConfig(query_parameters=query_params)
self.run_query(query, job_config=job_config)
def delete_multiple_rows_by_id(
self, table_id: str, id_column: str, ids: List[Any], id_value_type: str = "STRING"
) -> None:
"""
Deletes multiple rows from a table based on a list of IDs using parameterized query.
Args:
table_id: The ID of the table.
id_column: The name of the column containing the IDs.
ids: A list of ID values to delete.
id_value_type: The BigQuery type of the ID values.
"""
if not ids:
return # Nothing to delete
self._validate_column_name(id_column)
query = f"""
DELETE FROM `{self.project_id}.{self.dataset_id}.{table_id}`
WHERE `{id_column}` IN UNNEST(@ids);
"""
job_config = QueryJobConfig(
query_parameters=[
ArrayQueryParameter("ids", id_value_type, ids)
]
)
self.run_query(query, job_config=job_config)
def update_row_by_id(
self,
table_id: str,
id_column: str,
id_value: Any,
column_values: Dict[str, Any],
id_value_type: str = "STRING",
) -> None:
"""
Updates specific columns of a row identified by its ID using parameterized query.
Args:
table_id: The ID of the table.
id_column: The name of the column containing the ID.
id_value: The specific ID value of the row to update.
column_values: A dictionary where keys are column names and
values are the new values.
id_value_type: The BigQuery type of the ID value.
"""
if not column_values:
return
self._validate_column_name(id_column)
set_clauses = []
query_params_list: List[ScalarQueryParameter] = [
ScalarQueryParameter("id_value", id_value_type, id_value)
]
param_idx = 0
for col, current_value in column_values.items():
self._validate_column_name(col)
update_param_name = f"update_val_{param_idx}"
set_clauses.append(f"`{col}` = @{update_param_name}")
query_params_list.append(self._create_scalar_query_parameter(update_param_name, current_value))
param_idx += 1
if not set_clauses: # Should not happen if column_values is not empty
return
sets_str = ", ".join(set_clauses)
query = f"""
UPDATE `{self.project_id}.{self.dataset_id}.{table_id}`
SET {sets_str}
WHERE `{id_column}` = @id_value;
"""
job_config = QueryJobConfig(query_parameters=query_params_list)
self.run_query(query, job_config=job_config)
def get_all_rows(self, table_id: str) -> RowIterator:
"""
Retrieves all rows from the specified table.
Args:
table_id: The ID of the table.
Returns:
A RowIterator containing all rows in the table.
"""
query = f"SELECT * FROM `{self.project_id}.{self.dataset_id}.{table_id}`"
return self.run_query(query)
def delete_row_by_id(self, table_id: str, id_column: str, id_value: Any, id_value_type: str = "STRING") -> None:
"""
Deletes a single row from a table based on its ID using parameterized query.
Args:
table_id: The ID of the table.
id_column: The name of the column containing the ID.
id_value: The specific ID value of the row to delete.
id_value_type: The BigQuery type of the ID value.
"""
self._validate_column_name(id_column)
query = f"""
DELETE FROM `{self.project_id}.{self.dataset_id}.{table_id}`
WHERE `{id_column}` = @id_value;
"""
job_config = QueryJobConfig(
query_parameters=[
ScalarQueryParameter("id_value", id_value_type, id_value)
]
)
self.run_query(query, job_config=job_config)
@@ -0,0 +1,14 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
@@ -0,0 +1,77 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Service for interacting with Google Cloud Discovery Engine Engines."""
from typing import List
from google.cloud.discoveryengine_v1 import (
EngineServiceClient,
ListEnginesRequest,
)
from google.api_core.client_options import ClientOptions
from src.model.search import PROJECT_ID, Engine
LOCATIONS = [
"global",
"us",
]
class EngineService:
"""Provides methods to list Discovery Engine Engines."""
def get_all(self) -> List[Engine]:
"""
Retrieves all available Discovery Engines for the configured project
across specified locations.
Fetches engines from predefined locations ('global', 'us') within the
default collection.
Returns:
A list of Engine objects, each containing details about a
discovered engine. Returns an empty list if no engines are found
or if an error occurs during API calls.
Raises:
Prints error logs if API calls fail for a specific location.
"""
engines = []
for location in LOCATIONS:
client_options = (
ClientOptions(
api_endpoint=f"{location}-discoveryengine.googleapis.com"
)
if location != "global"
else None
)
# Create a client
client = EngineServiceClient(client_options=client_options)
list_engines = client.list_engines(
ListEnginesRequest(
parent=f"projects/{PROJECT_ID}/locations/{location}"
"/collections/default_collection"
)
)
for engine in list_engines.engines:
engines.append(
Engine(
name=engine.display_name,
engine_id=engine.name.split("/")[-1],
region=location,
)
)
return engines
@@ -0,0 +1,144 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Service for performing searches using Google Cloud Discovery Engine."""
from google.cloud.discoveryengine_v1 import SearchRequest, SearchServiceClient
from src.model.search import (
SearchApplication,
SearchResult,
SearchResultsWithSummary,
)
CONTENT_SEARCH_SPEC = SearchRequest.ContentSearchSpec(
snippet_spec=SearchRequest.ContentSearchSpec.SnippetSpec(
return_snippet=True
),
extractive_content_spec=SearchRequest.ContentSearchSpec.ExtractiveContentSpec(
max_extractive_answer_count=1
),
summary_spec=SearchRequest.ContentSearchSpec.SummarySpec(
summary_result_count=5,
include_citations=True,
ignore_adversarial_query=True,
ignore_non_summary_seeking_query=True,
model_spec=SearchRequest.ContentSearchSpec.SummarySpec.ModelSpec(
version="stable",
),
),
)
QUERY_EXPANSION_SPEC = SearchRequest.QueryExpansionSpec(
condition=SearchRequest.QueryExpansionSpec.Condition.AUTO,
)
SPELL_CORRECTION_SPEC = SearchRequest.SpellCorrectionSpec(
mode=SearchRequest.SpellCorrectionSpec.Mode.AUTO
)
class SearchService:
"""
Handles search operations using a configured
Discovery Engine Search Application.
Initializes with a specific SearchApplication configuration and provides
methods to perform searches against the corresponding engine.
"""
def __init__(self, search_application: SearchApplication):
"""
Initializes the SearchService.
Args:
search_application: The configuration object defining the target
search engine, location, and serving config.
"""
self.search_client = SearchServiceClient(
client_options=search_application.get_client_options()
)
self.serving_config = search_application.get_serving_config()
def search(self, term: str) -> SearchResultsWithSummary:
"""
Performs a search against the configured Discovery Engine.
Constructs a search request with predefined content specifications,
query expansion, and spell correction settings. Parses the response
to extract results, snippets, links, and a summary.
Args:
term: The search query string.
Returns:
A SearchResultsWithSummary object containing the search summary
and a list of SearchResult objects. Returns an empty list of
results and a default summary message if the search fails or
yields no results.
Raises:
Logs errors if the API call fails.
"""
request = SearchRequest(
serving_config=self.serving_config,
query=term,
page_size=10,
content_search_spec=CONTENT_SEARCH_SPEC,
query_expansion_spec=QUERY_EXPANSION_SPEC,
spell_correction_spec=SPELL_CORRECTION_SPEC,
)
data = self.search_client.search(request)
results = []
summary_text = (
data.summary.summary_text
if data.summary and data.summary.summary_text
else "No summary available"
)
# Process results
for r in data.results:
document = r.document
derived_data = document.derived_struct_data
gcs_link = derived_data.get("link").replace(
"gs://", "https://storage.cloud.google.com/"
)
# Extract snippet safely
snippets = derived_data.get("snippets", [])
snippet_text = (
snippets[0].get("snippet", "No snippet available")
if snippets
else "No snippet available"
)
extractive_answers = derived_data.get("extractive_answers", [])
content_text = (
extractive_answers[0].get("content", "No content available")
if extractive_answers
else "No content available"
)
# Map to SearchResult
mapped_result = SearchResult(
document_id=document.id,
title=derived_data.get("title", "Untitled"),
snippet=snippet_text,
link=gcs_link,
content=content_text,
)
results.append(mapped_result)
response_result = SearchResultsWithSummary(
summary=summary_text, results=results
)
return response_result
@@ -0,0 +1,119 @@
# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Service for managing the Search Application configuration
stored in BigQuery."""
from src.model.http_status import BadRequest
from src.model.search import SearchApplication
from src.repository.big_query import BigQueryRepository
SEARCH_APPLICATION_TABLE = "search_applications"
SEARCH_APPLICATION_TABLE_ID_COLUMN = "engine_id"
class SearchApplicationService:
"""Handles business logic for Search Application configuration."""
def __init__(self):
"""Initializes the service with a BigQuery repository."""
self.repository = BigQueryRepository()
def get(self):
"""
Retrieves the currently configured Search Application.
Assumes that there is at most one Search Application configuration
stored in the BigQuery table. If multiple rows exist, it returns
the last one processed.
Returns:
A SearchApplication object if found, otherwise None.
"""
search_application = None
results = self.repository.get_all_rows(SEARCH_APPLICATION_TABLE)
# Convert RowIterator to list to check if it's empty or get the last item
rows = list(results)
if rows:
# Assuming we take the last row if multiple exist, or the only one
search_application = SearchApplication.from_row(rows[-1])
return search_application
def create(
self, search_application: SearchApplication
) -> SearchApplication:
"""
Creates a new Search Application configuration in BigQuery.
Ensures that no configuration already exists before creating a new one.
Args:
search_application: The SearchApplication object to create.
Returns:
The created SearchApplication object.
Raises:
BadRequest: If a Search Application configuration already exists
for the project.
"""
if self.get():
raise BadRequest(
detail="Search Application for this project already exists"
)
schema_fields = SearchApplication.__schema__()
column_names = [field.name for field in schema_fields]
# Prepare values as a tuple, matching the order of schema_fields
# Ensure the order here matches the order in SearchApplication.__schema__
values_tuple = (search_application.engine_id, search_application.region)
self.repository.insert_row(
SEARCH_APPLICATION_TABLE,
column_names,
values_tuple,
)
return search_application
def update(self, engine_id: str, search_application: SearchApplication):
"""
Updates an existing Search Application configuration in BigQuery.
Identifies the row to update using the provided engine_id.
Args:
engine_id: The engine_id of the Search Application configuration
to update.
search_application: A SearchApplication object containing the
updated details (engine_id and region).
Returns:
The updated SearchApplication object.
Raises:
# Note: The underlying repository might raise exceptions on failure,
# which are not explicitly handled here.
# Consider adding error handling.
"""
update_dict = {
"engine_id": f'"{search_application.engine_id}"',
"region": f'"{search_application.region}"',
}
self.repository.update_row_by_id(
SEARCH_APPLICATION_TABLE,
SEARCH_APPLICATION_TABLE_ID_COLUMN,
engine_id,
update_dict,
)